5bf48a6686
Model: azalahmadkhan/Qwen2.5-3B-Instruct-GRPO-vanilla-G-8-novllm-75pct Source: Original Platform
39725 lines
1.4 MiB
39725 lines
1.4 MiB
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.75,
|
|
"eval_steps": 500,
|
|
"global_step": 1323,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1433.0,
|
|
"completions/max_terminated_length": 1433.0,
|
|
"completions/mean_length": 787.265625,
|
|
"completions/mean_terminated_length": 787.265625,
|
|
"completions/min_length": 415.0,
|
|
"completions/min_terminated_length": 415.0,
|
|
"entropy": 0.7912661358714104,
|
|
"epoch": 0.0005668934240362812,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.0005471410513564479,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0145,
|
|
"num_tokens": 58849.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 1,
|
|
"step_time": 55.261071532033384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/max_terminated_length": 1452.0,
|
|
"completions/mean_length": 712.390625,
|
|
"completions/mean_terminated_length": 712.390625,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.8053555078804493,
|
|
"epoch": 0.0011337868480725624,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.0005988259563309839,
|
|
"learning_rate": 9.994331065759637e-07,
|
|
"loss": -0.008,
|
|
"num_tokens": 116154.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 2,
|
|
"step_time": 56.21459399070591
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1385.0,
|
|
"completions/max_terminated_length": 1385.0,
|
|
"completions/mean_length": 705.8125,
|
|
"completions/mean_terminated_length": 705.8125,
|
|
"completions/min_length": 272.0,
|
|
"completions/min_terminated_length": 272.0,
|
|
"entropy": 0.9432813562452793,
|
|
"epoch": 0.0017006802721088435,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40625,
|
|
"kl": 0.0006910295905981911,
|
|
"learning_rate": 9.988662131519275e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 170830.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 3,
|
|
"step_time": 53.85885674227029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1959.0,
|
|
"completions/max_terminated_length": 1959.0,
|
|
"completions/mean_length": 756.921875,
|
|
"completions/mean_terminated_length": 756.921875,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 1.0130716506391764,
|
|
"epoch": 0.0022675736961451248,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2158203125,
|
|
"kl": 0.0007245079577842262,
|
|
"learning_rate": 9.982993197278912e-07,
|
|
"loss": -0.0027,
|
|
"num_tokens": 229481.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 4,
|
|
"step_time": 76.4772049030289
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1817.0,
|
|
"completions/max_terminated_length": 1817.0,
|
|
"completions/mean_length": 806.5,
|
|
"completions/mean_terminated_length": 806.5,
|
|
"completions/min_length": 342.0,
|
|
"completions/min_terminated_length": 342.0,
|
|
"entropy": 0.9002897161990404,
|
|
"epoch": 0.002834467120181406,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.000316619873046875,
|
|
"kl": 0.0006696486689179437,
|
|
"learning_rate": 9.977324263038547e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 290833.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 5,
|
|
"step_time": 70.04715261328965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1433.0,
|
|
"completions/max_terminated_length": 1433.0,
|
|
"completions/mean_length": 781.296875,
|
|
"completions/mean_terminated_length": 781.296875,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.6070616748183966,
|
|
"epoch": 0.003401360544217687,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.000568444460441242,
|
|
"learning_rate": 9.971655328798185e-07,
|
|
"loss": -0.0185,
|
|
"num_tokens": 351660.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 6,
|
|
"step_time": 55.606025882065296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1308.0,
|
|
"completions/max_terminated_length": 1308.0,
|
|
"completions/mean_length": 745.3125,
|
|
"completions/mean_terminated_length": 745.3125,
|
|
"completions/min_length": 459.0,
|
|
"completions/min_terminated_length": 459.0,
|
|
"entropy": 0.8628565222024918,
|
|
"epoch": 0.003968253968253968,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.0006363487227645237,
|
|
"learning_rate": 9.965986394557822e-07,
|
|
"loss": 0.0025,
|
|
"num_tokens": 412128.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 7,
|
|
"step_time": 51.90570120885968
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1151.0,
|
|
"completions/max_terminated_length": 1151.0,
|
|
"completions/mean_length": 666.1875,
|
|
"completions/mean_terminated_length": 666.1875,
|
|
"completions/min_length": 238.0,
|
|
"completions/min_terminated_length": 238.0,
|
|
"entropy": 0.782693337649107,
|
|
"epoch": 0.0045351473922902496,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.39453125,
|
|
"kl": 0.0006672325453109806,
|
|
"learning_rate": 9.96031746031746e-07,
|
|
"loss": -0.0049,
|
|
"num_tokens": 466628.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 8,
|
|
"step_time": 44.9562089079991
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1280.0,
|
|
"completions/max_terminated_length": 1280.0,
|
|
"completions/mean_length": 789.53125,
|
|
"completions/mean_terminated_length": 789.53125,
|
|
"completions/min_length": 453.0,
|
|
"completions/min_terminated_length": 453.0,
|
|
"entropy": 0.7709263563156128,
|
|
"epoch": 0.00510204081632653,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.20703125,
|
|
"kl": 0.000592712800425943,
|
|
"learning_rate": 9.954648526077097e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 528558.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 9,
|
|
"step_time": 50.013049360364676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1361.0,
|
|
"completions/max_terminated_length": 1361.0,
|
|
"completions/mean_length": 834.765625,
|
|
"completions/mean_terminated_length": 834.765625,
|
|
"completions/min_length": 501.0,
|
|
"completions/min_terminated_length": 501.0,
|
|
"entropy": 0.8164468556642532,
|
|
"epoch": 0.005668934240362812,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.0006466719878517324,
|
|
"learning_rate": 9.948979591836735e-07,
|
|
"loss": 0.0143,
|
|
"num_tokens": 591359.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 10,
|
|
"step_time": 53.13434873986989
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1307.0,
|
|
"completions/max_terminated_length": 1307.0,
|
|
"completions/mean_length": 819.125,
|
|
"completions/mean_terminated_length": 819.125,
|
|
"completions/min_length": 415.0,
|
|
"completions/min_terminated_length": 415.0,
|
|
"entropy": 0.8207502197474241,
|
|
"epoch": 0.006235827664399093,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.35546875,
|
|
"kl": 0.0006157456555229146,
|
|
"learning_rate": 9.94331065759637e-07,
|
|
"loss": -0.0009,
|
|
"num_tokens": 653527.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 11,
|
|
"step_time": 51.11214052326977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1524.0,
|
|
"completions/max_terminated_length": 1524.0,
|
|
"completions/mean_length": 818.0625,
|
|
"completions/mean_terminated_length": 818.0625,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.7241049911826849,
|
|
"epoch": 0.006802721088435374,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.0005700781130144605,
|
|
"learning_rate": 9.937641723356008e-07,
|
|
"loss": -0.0101,
|
|
"num_tokens": 715875.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 12,
|
|
"step_time": 59.11056109983474
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1607.0,
|
|
"completions/max_terminated_length": 1607.0,
|
|
"completions/mean_length": 912.9375,
|
|
"completions/mean_terminated_length": 912.9375,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.7451053559780121,
|
|
"epoch": 0.007369614512471655,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1845703125,
|
|
"kl": 0.000535154516910552,
|
|
"learning_rate": 9.931972789115645e-07,
|
|
"loss": -0.0058,
|
|
"num_tokens": 785487.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 13,
|
|
"step_time": 61.979948569089174
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1222.0,
|
|
"completions/max_terminated_length": 1222.0,
|
|
"completions/mean_length": 743.3125,
|
|
"completions/mean_terminated_length": 743.3125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.8540959488600492,
|
|
"epoch": 0.007936507936507936,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.333984375,
|
|
"kl": 0.0006348256429191679,
|
|
"learning_rate": 9.926303854875283e-07,
|
|
"loss": 0.0172,
|
|
"num_tokens": 844387.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 14,
|
|
"step_time": 48.2400392210111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2454.0,
|
|
"completions/max_terminated_length": 2454.0,
|
|
"completions/mean_length": 789.453125,
|
|
"completions/mean_terminated_length": 789.453125,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.6644097231328487,
|
|
"epoch": 0.008503401360544218,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.0006174067239044234,
|
|
"learning_rate": 9.92063492063492e-07,
|
|
"loss": -0.0117,
|
|
"num_tokens": 905664.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 15,
|
|
"step_time": 97.98640740290284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1459.0,
|
|
"completions/max_terminated_length": 1459.0,
|
|
"completions/mean_length": 778.0625,
|
|
"completions/mean_terminated_length": 778.0625,
|
|
"completions/min_length": 400.0,
|
|
"completions/min_terminated_length": 400.0,
|
|
"entropy": 0.8444338124245405,
|
|
"epoch": 0.009070294784580499,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.0007286928284884198,
|
|
"learning_rate": 9.914965986394558e-07,
|
|
"loss": 0.0031,
|
|
"num_tokens": 965404.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 16,
|
|
"step_time": 56.34489590395242
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1568.0,
|
|
"completions/max_terminated_length": 1568.0,
|
|
"completions/mean_length": 801.015625,
|
|
"completions/mean_terminated_length": 801.015625,
|
|
"completions/min_length": 224.0,
|
|
"completions/min_terminated_length": 224.0,
|
|
"entropy": 0.967936534434557,
|
|
"epoch": 0.00963718820861678,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.0007015223854978103,
|
|
"learning_rate": 9.909297052154193e-07,
|
|
"loss": -0.0194,
|
|
"num_tokens": 1027645.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 17,
|
|
"step_time": 61.04807883966714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2414.0,
|
|
"completions/max_terminated_length": 2414.0,
|
|
"completions/mean_length": 824.53125,
|
|
"completions/mean_terminated_length": 824.53125,
|
|
"completions/min_length": 306.0,
|
|
"completions/min_terminated_length": 306.0,
|
|
"entropy": 0.6849733348935843,
|
|
"epoch": 0.01020408163265306,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.306640625,
|
|
"kl": 0.0005724504480895121,
|
|
"learning_rate": 9.90362811791383e-07,
|
|
"loss": -0.0081,
|
|
"num_tokens": 1090127.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 18,
|
|
"step_time": 95.73333193920553
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1571.0,
|
|
"completions/max_terminated_length": 1571.0,
|
|
"completions/mean_length": 725.53125,
|
|
"completions/mean_terminated_length": 725.53125,
|
|
"completions/min_length": 402.0,
|
|
"completions/min_terminated_length": 402.0,
|
|
"entropy": 1.071934748440981,
|
|
"epoch": 0.010770975056689343,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.0007020830926194321,
|
|
"learning_rate": 9.897959183673468e-07,
|
|
"loss": -0.0168,
|
|
"num_tokens": 1146105.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 19,
|
|
"step_time": 61.04049335233867
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1878.0,
|
|
"completions/max_terminated_length": 1878.0,
|
|
"completions/mean_length": 790.359375,
|
|
"completions/mean_terminated_length": 790.359375,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.7894083205610514,
|
|
"epoch": 0.011337868480725623,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.365234375,
|
|
"kl": 0.000633090885457932,
|
|
"learning_rate": 9.892290249433106e-07,
|
|
"loss": 0.0443,
|
|
"num_tokens": 1206824.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 20,
|
|
"step_time": 73.91199010517448
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1718.0,
|
|
"completions/max_terminated_length": 1718.0,
|
|
"completions/mean_length": 854.046875,
|
|
"completions/mean_terminated_length": 854.046875,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.7625202462077141,
|
|
"epoch": 0.011904761904761904,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.201171875,
|
|
"kl": 0.0006028276457072934,
|
|
"learning_rate": 9.886621315192743e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 1274483.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 21,
|
|
"step_time": 68.21598854195327
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1261.0,
|
|
"completions/max_terminated_length": 1261.0,
|
|
"completions/mean_length": 692.296875,
|
|
"completions/mean_terminated_length": 692.296875,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"entropy": 0.8724542260169983,
|
|
"epoch": 0.012471655328798186,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.458984375,
|
|
"kl": 0.0008029471355257556,
|
|
"learning_rate": 9.88095238095238e-07,
|
|
"loss": -0.0082,
|
|
"num_tokens": 1337598.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5883605480194092,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 22,
|
|
"step_time": 52.92388445697725
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1279.0,
|
|
"completions/mean_length": 820.921875,
|
|
"completions/mean_terminated_length": 768.9365234375,
|
|
"completions/min_length": 206.0,
|
|
"completions/min_terminated_length": 206.0,
|
|
"entropy": 0.7689987067133188,
|
|
"epoch": 0.013038548752834467,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2060546875,
|
|
"kl": 0.0006323507077468093,
|
|
"learning_rate": 9.875283446712018e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 1401201.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 23,
|
|
"step_time": 181.87214892730117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1296.0,
|
|
"completions/max_terminated_length": 1296.0,
|
|
"completions/mean_length": 679.171875,
|
|
"completions/mean_terminated_length": 679.171875,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 1.0151823163032532,
|
|
"epoch": 0.013605442176870748,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.34765625,
|
|
"kl": 0.0006834342384536285,
|
|
"learning_rate": 9.869614512471656e-07,
|
|
"loss": -0.0117,
|
|
"num_tokens": 1455220.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 24,
|
|
"step_time": 50.659633063711226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1289.0,
|
|
"completions/max_terminated_length": 1289.0,
|
|
"completions/mean_length": 779.484375,
|
|
"completions/mean_terminated_length": 779.484375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.8662557099014521,
|
|
"epoch": 0.01417233560090703,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.000492095947265625,
|
|
"kl": 0.0006726052361045731,
|
|
"learning_rate": 9.86394557823129e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1515651.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 25,
|
|
"step_time": 50.44343617465347
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1112.0,
|
|
"completions/max_terminated_length": 1112.0,
|
|
"completions/mean_length": 746.453125,
|
|
"completions/mean_terminated_length": 746.453125,
|
|
"completions/min_length": 393.0,
|
|
"completions/min_terminated_length": 393.0,
|
|
"entropy": 0.8149989545345306,
|
|
"epoch": 0.01473922902494331,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.0006371798408508766,
|
|
"learning_rate": 9.858276643990928e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 1572696.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 26,
|
|
"step_time": 44.474265119060874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1443.0,
|
|
"completions/max_terminated_length": 1443.0,
|
|
"completions/mean_length": 715.890625,
|
|
"completions/mean_terminated_length": 715.890625,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 1.095861166715622,
|
|
"epoch": 0.015306122448979591,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00019359588623046875,
|
|
"kl": 0.0007180045668064849,
|
|
"learning_rate": 9.852607709750566e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1629089.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 27,
|
|
"step_time": 57.39974437747151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1605.0,
|
|
"completions/max_terminated_length": 1605.0,
|
|
"completions/mean_length": 718.109375,
|
|
"completions/mean_terminated_length": 718.109375,
|
|
"completions/min_length": 193.0,
|
|
"completions/min_terminated_length": 193.0,
|
|
"entropy": 0.826047021895647,
|
|
"epoch": 0.015873015873015872,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.0006650087561865803,
|
|
"learning_rate": 9.846938775510203e-07,
|
|
"loss": -0.0281,
|
|
"num_tokens": 1684944.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 28,
|
|
"step_time": 64.38864888530225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1537.0,
|
|
"completions/max_terminated_length": 1537.0,
|
|
"completions/mean_length": 826.625,
|
|
"completions/mean_terminated_length": 826.625,
|
|
"completions/min_length": 23.0,
|
|
"completions/min_terminated_length": 23.0,
|
|
"entropy": 0.8258813861757517,
|
|
"epoch": 0.016439909297052153,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.263671875,
|
|
"kl": 0.0006447574651247123,
|
|
"learning_rate": 9.84126984126984e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 1750912.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 29,
|
|
"step_time": 60.81136659439653
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1433.0,
|
|
"completions/max_terminated_length": 1433.0,
|
|
"completions/mean_length": 775.40625,
|
|
"completions/mean_terminated_length": 775.40625,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.8379686512053013,
|
|
"epoch": 0.017006802721088437,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.24609375,
|
|
"kl": 0.0006821296829002677,
|
|
"learning_rate": 9.835600907029478e-07,
|
|
"loss": -0.0105,
|
|
"num_tokens": 1812026.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 30,
|
|
"step_time": 57.01410628668964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2075.0,
|
|
"completions/max_terminated_length": 2075.0,
|
|
"completions/mean_length": 785.953125,
|
|
"completions/mean_terminated_length": 785.953125,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.915758527815342,
|
|
"epoch": 0.017573696145124718,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.0006477457645814866,
|
|
"learning_rate": 9.829931972789116e-07,
|
|
"loss": 0.0075,
|
|
"num_tokens": 1872023.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 31,
|
|
"step_time": 81.80097121186554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1282.0,
|
|
"completions/max_terminated_length": 1282.0,
|
|
"completions/mean_length": 792.578125,
|
|
"completions/mean_terminated_length": 792.578125,
|
|
"completions/min_length": 275.0,
|
|
"completions/min_terminated_length": 275.0,
|
|
"entropy": 0.9764898531138897,
|
|
"epoch": 0.018140589569160998,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.330078125,
|
|
"kl": 0.000687534624375985,
|
|
"learning_rate": 9.824263038548753e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 1935372.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 32,
|
|
"step_time": 51.07413536589593
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1377.0,
|
|
"completions/max_terminated_length": 1377.0,
|
|
"completions/mean_length": 720.671875,
|
|
"completions/mean_terminated_length": 720.671875,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.8088947497308254,
|
|
"epoch": 0.01870748299319728,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.0006506219633592991,
|
|
"learning_rate": 9.818594104308389e-07,
|
|
"loss": 0.008,
|
|
"num_tokens": 1990167.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 33,
|
|
"step_time": 53.4132538670674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1718.0,
|
|
"completions/max_terminated_length": 1718.0,
|
|
"completions/mean_length": 785.0625,
|
|
"completions/mean_terminated_length": 785.0625,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.6793034747242928,
|
|
"epoch": 0.01927437641723356,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.314453125,
|
|
"kl": 0.0005896855145692825,
|
|
"learning_rate": 9.812925170068026e-07,
|
|
"loss": -0.0068,
|
|
"num_tokens": 2050579.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 34,
|
|
"step_time": 66.1939322706312
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1336.0,
|
|
"completions/max_terminated_length": 1336.0,
|
|
"completions/mean_length": 786.03125,
|
|
"completions/mean_terminated_length": 786.03125,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.7928113117814064,
|
|
"epoch": 0.01984126984126984,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.0006361865598591976,
|
|
"learning_rate": 9.807256235827664e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 2112461.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 35,
|
|
"step_time": 52.14811750501394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1769.0,
|
|
"completions/max_terminated_length": 1769.0,
|
|
"completions/mean_length": 745.390625,
|
|
"completions/mean_terminated_length": 745.390625,
|
|
"completions/min_length": 210.0,
|
|
"completions/min_terminated_length": 210.0,
|
|
"entropy": 0.9053811207413673,
|
|
"epoch": 0.02040816326530612,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00020885467529296875,
|
|
"kl": 0.0006936117097211536,
|
|
"learning_rate": 9.801587301587301e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2171030.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 36,
|
|
"step_time": 68.92622819449753
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1255.0,
|
|
"completions/max_terminated_length": 1255.0,
|
|
"completions/mean_length": 770.296875,
|
|
"completions/mean_terminated_length": 770.296875,
|
|
"completions/min_length": 420.0,
|
|
"completions/min_terminated_length": 420.0,
|
|
"entropy": 0.6799955926835537,
|
|
"epoch": 0.020975056689342405,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.44921875,
|
|
"kl": 0.0005374719130486483,
|
|
"learning_rate": 9.795918367346939e-07,
|
|
"loss": -0.0226,
|
|
"num_tokens": 2231729.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7867957949638367,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 37,
|
|
"step_time": 49.94026922620833
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1254.0,
|
|
"completions/max_terminated_length": 1254.0,
|
|
"completions/mean_length": 778.9375,
|
|
"completions/mean_terminated_length": 778.9375,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7798807322978973,
|
|
"epoch": 0.021541950113378686,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.24609375,
|
|
"kl": 0.0005999129698466277,
|
|
"learning_rate": 9.790249433106576e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 2291117.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 38,
|
|
"step_time": 49.6709918724373
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1569.0,
|
|
"completions/max_terminated_length": 1569.0,
|
|
"completions/mean_length": 762.765625,
|
|
"completions/mean_terminated_length": 762.765625,
|
|
"completions/min_length": 436.0,
|
|
"completions/min_terminated_length": 436.0,
|
|
"entropy": 1.0847668685019016,
|
|
"epoch": 0.022108843537414966,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.0007197252125479281,
|
|
"learning_rate": 9.784580498866214e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 2350870.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 39,
|
|
"step_time": 61.75452910736203
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1609.0,
|
|
"completions/max_terminated_length": 1609.0,
|
|
"completions/mean_length": 710.671875,
|
|
"completions/mean_terminated_length": 710.671875,
|
|
"completions/min_length": 216.0,
|
|
"completions/min_terminated_length": 216.0,
|
|
"entropy": 1.0219030529260635,
|
|
"epoch": 0.022675736961451247,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00030517578125,
|
|
"kl": 0.0007319577052840032,
|
|
"learning_rate": 9.778911564625851e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2411049.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 40,
|
|
"step_time": 64.2758515747264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1166.0,
|
|
"completions/max_terminated_length": 1166.0,
|
|
"completions/mean_length": 692.0,
|
|
"completions/mean_terminated_length": 692.0,
|
|
"completions/min_length": 23.0,
|
|
"completions/min_terminated_length": 23.0,
|
|
"entropy": 0.9765904024243355,
|
|
"epoch": 0.023242630385487528,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.447265625,
|
|
"kl": 0.0007533810530730989,
|
|
"learning_rate": 9.773242630385487e-07,
|
|
"loss": -0.018,
|
|
"num_tokens": 2465801.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 41,
|
|
"step_time": 45.983314889483154
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1518.0,
|
|
"completions/max_terminated_length": 1518.0,
|
|
"completions/mean_length": 939.328125,
|
|
"completions/mean_terminated_length": 939.328125,
|
|
"completions/min_length": 475.0,
|
|
"completions/min_terminated_length": 475.0,
|
|
"entropy": 0.8025281708687544,
|
|
"epoch": 0.023809523809523808,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.0006058815342839807,
|
|
"learning_rate": 9.767573696145124e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 2537286.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 42,
|
|
"step_time": 61.44131015893072
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1457.0,
|
|
"completions/max_terminated_length": 1457.0,
|
|
"completions/mean_length": 734.328125,
|
|
"completions/mean_terminated_length": 734.328125,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.739720705896616,
|
|
"epoch": 0.024376417233560092,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.42578125,
|
|
"kl": 0.0006070303206797689,
|
|
"learning_rate": 9.761904761904762e-07,
|
|
"loss": 0.0228,
|
|
"num_tokens": 2594475.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 43,
|
|
"step_time": 57.56407857872546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1369.0,
|
|
"completions/max_terminated_length": 1369.0,
|
|
"completions/mean_length": 799.140625,
|
|
"completions/mean_terminated_length": 799.140625,
|
|
"completions/min_length": 77.0,
|
|
"completions/min_terminated_length": 77.0,
|
|
"entropy": 0.8235787898302078,
|
|
"epoch": 0.024943310657596373,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.0006626893518841825,
|
|
"learning_rate": 9.7562358276644e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 2655684.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 44,
|
|
"step_time": 53.67061932384968
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1431.0,
|
|
"completions/max_terminated_length": 1431.0,
|
|
"completions/mean_length": 768.5,
|
|
"completions/mean_terminated_length": 768.5,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 1.153789233416319,
|
|
"epoch": 0.025510204081632654,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.000347137451171875,
|
|
"kl": 0.0007488102091883775,
|
|
"learning_rate": 9.750566893424036e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2716276.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 45,
|
|
"step_time": 56.18650230206549
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1416.0,
|
|
"completions/max_terminated_length": 1416.0,
|
|
"completions/mean_length": 771.671875,
|
|
"completions/mean_terminated_length": 771.671875,
|
|
"completions/min_length": 156.0,
|
|
"completions/min_terminated_length": 156.0,
|
|
"entropy": 0.7739651259034872,
|
|
"epoch": 0.026077097505668934,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0002288818359375,
|
|
"kl": 0.0006726973533659475,
|
|
"learning_rate": 9.744897959183674e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2776359.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 46,
|
|
"step_time": 55.43411583080888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 753.828125,
|
|
"completions/mean_terminated_length": 753.828125,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.7858696226030588,
|
|
"epoch": 0.026643990929705215,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.15625,
|
|
"kl": 0.0006061669355403865,
|
|
"learning_rate": 9.739229024943311e-07,
|
|
"loss": 0.0046,
|
|
"num_tokens": 2837924.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 47,
|
|
"step_time": 50.96035030670464
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1856.0,
|
|
"completions/max_terminated_length": 1856.0,
|
|
"completions/mean_length": 832.484375,
|
|
"completions/mean_terminated_length": 832.484375,
|
|
"completions/min_length": 113.0,
|
|
"completions/min_terminated_length": 113.0,
|
|
"entropy": 0.6921264715492725,
|
|
"epoch": 0.027210884353741496,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.314453125,
|
|
"kl": 0.0005807866637042025,
|
|
"learning_rate": 9.733560090702947e-07,
|
|
"loss": 0.0138,
|
|
"num_tokens": 2900523.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 48,
|
|
"step_time": 72.53957574628294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1209.0,
|
|
"completions/max_terminated_length": 1209.0,
|
|
"completions/mean_length": 690.640625,
|
|
"completions/mean_terminated_length": 690.640625,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.9992184825241566,
|
|
"epoch": 0.027777777777777776,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.0007191828335635364,
|
|
"learning_rate": 9.727891156462584e-07,
|
|
"loss": 0.0092,
|
|
"num_tokens": 2954092.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 49,
|
|
"step_time": 47.970756710506976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1233.0,
|
|
"completions/max_terminated_length": 1233.0,
|
|
"completions/mean_length": 763.9375,
|
|
"completions/mean_terminated_length": 763.9375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.8453336097300053,
|
|
"epoch": 0.02834467120181406,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.0006714449973515002,
|
|
"learning_rate": 9.722222222222222e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 3013360.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 50,
|
|
"step_time": 49.39462559018284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1286.0,
|
|
"completions/max_terminated_length": 1286.0,
|
|
"completions/mean_length": 784.921875,
|
|
"completions/mean_terminated_length": 784.921875,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.7711440324783325,
|
|
"epoch": 0.02891156462585034,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.0006841422764409799,
|
|
"learning_rate": 9.71655328798186e-07,
|
|
"loss": -0.0161,
|
|
"num_tokens": 3077283.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 51,
|
|
"step_time": 54.76244755461812
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2038.0,
|
|
"completions/max_terminated_length": 2038.0,
|
|
"completions/mean_length": 824.328125,
|
|
"completions/mean_terminated_length": 824.328125,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.8741871435195208,
|
|
"epoch": 0.02947845804988662,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00022220611572265625,
|
|
"kl": 0.0006717530995956622,
|
|
"learning_rate": 9.710884353741497e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3139384.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 52,
|
|
"step_time": 81.03633813653141
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1312.0,
|
|
"completions/max_terminated_length": 1312.0,
|
|
"completions/mean_length": 685.78125,
|
|
"completions/mean_terminated_length": 685.78125,
|
|
"completions/min_length": 385.0,
|
|
"completions/min_terminated_length": 385.0,
|
|
"entropy": 1.008056415244937,
|
|
"epoch": 0.030045351473922902,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2099609375,
|
|
"kl": 0.0008027284893614706,
|
|
"learning_rate": 9.705215419501134e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 3193314.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 53,
|
|
"step_time": 51.80523566156626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1396.0,
|
|
"completions/max_terminated_length": 1396.0,
|
|
"completions/mean_length": 847.921875,
|
|
"completions/mean_terminated_length": 847.921875,
|
|
"completions/min_length": 383.0,
|
|
"completions/min_terminated_length": 383.0,
|
|
"entropy": 0.8684836961328983,
|
|
"epoch": 0.030612244897959183,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1533203125,
|
|
"kl": 0.0006654198623436969,
|
|
"learning_rate": 9.69954648526077e-07,
|
|
"loss": 0.0056,
|
|
"num_tokens": 3258621.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 54,
|
|
"step_time": 55.535200772807
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 714.484375,
|
|
"completions/mean_terminated_length": 714.484375,
|
|
"completions/min_length": 184.0,
|
|
"completions/min_terminated_length": 184.0,
|
|
"entropy": 0.9482756368815899,
|
|
"epoch": 0.031179138321995464,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.16796875,
|
|
"kl": 0.0007149345219659153,
|
|
"learning_rate": 9.693877551020407e-07,
|
|
"loss": 0.0063,
|
|
"num_tokens": 3315644.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 55,
|
|
"step_time": 51.57503576390445
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1504.0,
|
|
"completions/max_terminated_length": 1504.0,
|
|
"completions/mean_length": 716.9375,
|
|
"completions/mean_terminated_length": 716.9375,
|
|
"completions/min_length": 242.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"entropy": 0.92583417147398,
|
|
"epoch": 0.031746031746031744,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.42578125,
|
|
"kl": 0.0007983353825693484,
|
|
"learning_rate": 9.688208616780045e-07,
|
|
"loss": -0.0053,
|
|
"num_tokens": 3373208.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 56,
|
|
"step_time": 59.63824711274356
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1363.0,
|
|
"completions/max_terminated_length": 1363.0,
|
|
"completions/mean_length": 726.875,
|
|
"completions/mean_terminated_length": 726.875,
|
|
"completions/min_length": 370.0,
|
|
"completions/min_terminated_length": 370.0,
|
|
"entropy": 0.7292650081217289,
|
|
"epoch": 0.03231292517006803,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00052642822265625,
|
|
"kl": 0.0007186582824942889,
|
|
"learning_rate": 9.682539682539682e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3432760.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 57,
|
|
"step_time": 55.06257279776037
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1301.0,
|
|
"completions/max_terminated_length": 1301.0,
|
|
"completions/mean_length": 712.390625,
|
|
"completions/mean_terminated_length": 712.390625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.7277219910174608,
|
|
"epoch": 0.032879818594104306,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6328125,
|
|
"kl": 0.0006634861638303846,
|
|
"learning_rate": 9.67687074829932e-07,
|
|
"loss": 0.0349,
|
|
"num_tokens": 3487625.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.7003666162490845,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 58,
|
|
"step_time": 51.896855810657144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1332.0,
|
|
"completions/max_terminated_length": 1332.0,
|
|
"completions/mean_length": 672.078125,
|
|
"completions/mean_terminated_length": 672.078125,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.9556263759732246,
|
|
"epoch": 0.03344671201814059,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.0008063717141340021,
|
|
"learning_rate": 9.671201814058957e-07,
|
|
"loss": -0.0271,
|
|
"num_tokens": 3541342.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4917473793029785,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 59,
|
|
"step_time": 53.21447585616261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1470.0,
|
|
"completions/mean_length": 816.28125,
|
|
"completions/mean_terminated_length": 764.2222900390625,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 0.9738425239920616,
|
|
"epoch": 0.034013605442176874,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.0007435131665260997,
|
|
"learning_rate": 9.665532879818595e-07,
|
|
"loss": -0.0248,
|
|
"num_tokens": 3604608.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 60,
|
|
"step_time": 182.24652972444892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2197.0,
|
|
"completions/max_terminated_length": 2197.0,
|
|
"completions/mean_length": 811.84375,
|
|
"completions/mean_terminated_length": 811.84375,
|
|
"completions/min_length": 342.0,
|
|
"completions/min_terminated_length": 342.0,
|
|
"entropy": 0.8543225675821304,
|
|
"epoch": 0.03458049886621315,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.21484375,
|
|
"kl": 0.0007339417061302811,
|
|
"learning_rate": 9.65986394557823e-07,
|
|
"loss": 0.0047,
|
|
"num_tokens": 3667670.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 61,
|
|
"step_time": 86.90420265123248
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1365.0,
|
|
"completions/max_terminated_length": 1365.0,
|
|
"completions/mean_length": 659.34375,
|
|
"completions/mean_terminated_length": 659.34375,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.8573559112846851,
|
|
"epoch": 0.035147392290249435,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"kl": 0.0006897826133354101,
|
|
"learning_rate": 9.654195011337868e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 3718516.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 62,
|
|
"step_time": 53.312431974336505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1478.0,
|
|
"completions/max_terminated_length": 1478.0,
|
|
"completions/mean_length": 730.34375,
|
|
"completions/mean_terminated_length": 730.34375,
|
|
"completions/min_length": 258.0,
|
|
"completions/min_terminated_length": 258.0,
|
|
"entropy": 0.9046593047678471,
|
|
"epoch": 0.03571428571428571,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.0007334729616559343,
|
|
"learning_rate": 9.648526077097505e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 3775250.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6666666865348816,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 63,
|
|
"step_time": 56.92630753200501
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1827.0,
|
|
"completions/max_terminated_length": 1827.0,
|
|
"completions/mean_length": 793.21875,
|
|
"completions/mean_terminated_length": 793.21875,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 0.7975286822766066,
|
|
"epoch": 0.036281179138321996,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.0006675975000689505,
|
|
"learning_rate": 9.642857142857142e-07,
|
|
"loss": -0.0224,
|
|
"num_tokens": 3838456.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 64,
|
|
"step_time": 72.07502091303468
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1546.0,
|
|
"completions/max_terminated_length": 1546.0,
|
|
"completions/mean_length": 746.296875,
|
|
"completions/mean_terminated_length": 746.296875,
|
|
"completions/min_length": 248.0,
|
|
"completions/min_terminated_length": 248.0,
|
|
"entropy": 0.9710622020065784,
|
|
"epoch": 0.036848072562358274,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.0007247600697155576,
|
|
"learning_rate": 9.63718820861678e-07,
|
|
"loss": 0.0081,
|
|
"num_tokens": 3897779.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 65,
|
|
"step_time": 61.013609156012535
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1482.0,
|
|
"completions/max_terminated_length": 1482.0,
|
|
"completions/mean_length": 872.578125,
|
|
"completions/mean_terminated_length": 872.578125,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 0.6040001548826694,
|
|
"epoch": 0.03741496598639456,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.251953125,
|
|
"kl": 0.0006071323223295622,
|
|
"learning_rate": 9.631519274376417e-07,
|
|
"loss": 0.0031,
|
|
"num_tokens": 3966128.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 66,
|
|
"step_time": 59.130399737507105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1829.0,
|
|
"completions/max_terminated_length": 1829.0,
|
|
"completions/mean_length": 741.125,
|
|
"completions/mean_terminated_length": 741.125,
|
|
"completions/min_length": 82.0,
|
|
"completions/min_terminated_length": 82.0,
|
|
"entropy": 0.7512783892452717,
|
|
"epoch": 0.03798185941043084,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.0006990785877860617,
|
|
"learning_rate": 9.625850340136053e-07,
|
|
"loss": -0.0309,
|
|
"num_tokens": 4024048.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 67,
|
|
"step_time": 71.48969585448503
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1737.0,
|
|
"completions/max_terminated_length": 1737.0,
|
|
"completions/mean_length": 935.09375,
|
|
"completions/mean_terminated_length": 935.09375,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.6651214510202408,
|
|
"epoch": 0.03854875283446712,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.000569740826904308,
|
|
"learning_rate": 9.62018140589569e-07,
|
|
"loss": -0.0019,
|
|
"num_tokens": 4094222.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 68,
|
|
"step_time": 68.3381945444271
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1524.0,
|
|
"completions/mean_length": 856.453125,
|
|
"completions/mean_terminated_length": 805.0317993164062,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.7249712217599154,
|
|
"epoch": 0.0391156462585034,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.0006563397655554581,
|
|
"learning_rate": 9.614512471655328e-07,
|
|
"loss": -0.0054,
|
|
"num_tokens": 4160611.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 69,
|
|
"step_time": 181.34099934529513
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1827.0,
|
|
"completions/max_terminated_length": 1827.0,
|
|
"completions/mean_length": 763.640625,
|
|
"completions/mean_terminated_length": 763.640625,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.9422737546265125,
|
|
"epoch": 0.03968253968253968,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.0008529020524292719,
|
|
"learning_rate": 9.608843537414965e-07,
|
|
"loss": 0.0356,
|
|
"num_tokens": 4219740.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5422958731651306,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 70,
|
|
"step_time": 71.48918853607029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1177.0,
|
|
"completions/max_terminated_length": 1177.0,
|
|
"completions/mean_length": 629.34375,
|
|
"completions/mean_terminated_length": 629.34375,
|
|
"completions/min_length": 295.0,
|
|
"completions/min_terminated_length": 295.0,
|
|
"entropy": 0.7320157140493393,
|
|
"epoch": 0.040249433106575964,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.291015625,
|
|
"kl": 0.0007003796126809902,
|
|
"learning_rate": 9.603174603174603e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 4269650.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 71,
|
|
"step_time": 46.33684809971601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1225.0,
|
|
"completions/max_terminated_length": 1225.0,
|
|
"completions/mean_length": 607.6875,
|
|
"completions/mean_terminated_length": 607.6875,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.8365393560379744,
|
|
"epoch": 0.04081632653061224,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.0007384430427919142,
|
|
"learning_rate": 9.59750566893424e-07,
|
|
"loss": 0.0073,
|
|
"num_tokens": 4316630.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 72,
|
|
"step_time": 47.365977444685996
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1606.0,
|
|
"completions/max_terminated_length": 1606.0,
|
|
"completions/mean_length": 766.859375,
|
|
"completions/mean_terminated_length": 766.859375,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.7943666074424982,
|
|
"epoch": 0.041383219954648526,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.0006592419031221652,
|
|
"learning_rate": 9.591836734693876e-07,
|
|
"loss": 0.0108,
|
|
"num_tokens": 4383725.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 73,
|
|
"step_time": 70.14359752461314
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1080.0,
|
|
"completions/max_terminated_length": 1080.0,
|
|
"completions/mean_length": 679.75,
|
|
"completions/mean_terminated_length": 679.75,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 1.0987933464348316,
|
|
"epoch": 0.04195011337868481,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.0008692541960044764,
|
|
"learning_rate": 9.586167800453513e-07,
|
|
"loss": -0.0024,
|
|
"num_tokens": 4436613.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 74,
|
|
"step_time": 44.69452762231231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1387.0,
|
|
"completions/max_terminated_length": 1387.0,
|
|
"completions/mean_length": 829.5625,
|
|
"completions/mean_terminated_length": 829.5625,
|
|
"completions/min_length": 411.0,
|
|
"completions/min_terminated_length": 411.0,
|
|
"entropy": 0.6923223361372948,
|
|
"epoch": 0.04251700680272109,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.0006142441598058213,
|
|
"learning_rate": 9.58049886621315e-07,
|
|
"loss": -0.0138,
|
|
"num_tokens": 4499945.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 75,
|
|
"step_time": 56.41232482995838
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1258.0,
|
|
"completions/max_terminated_length": 1258.0,
|
|
"completions/mean_length": 701.484375,
|
|
"completions/mean_terminated_length": 701.484375,
|
|
"completions/min_length": 264.0,
|
|
"completions/min_terminated_length": 264.0,
|
|
"entropy": 0.8171689808368683,
|
|
"epoch": 0.04308390022675737,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30859375,
|
|
"kl": 0.000773871553974459,
|
|
"learning_rate": 9.574829931972788e-07,
|
|
"loss": 0.0193,
|
|
"num_tokens": 4553632.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 76,
|
|
"step_time": 48.506453486159444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1288.0,
|
|
"completions/max_terminated_length": 1288.0,
|
|
"completions/mean_length": 779.9375,
|
|
"completions/mean_terminated_length": 779.9375,
|
|
"completions/min_length": 399.0,
|
|
"completions/min_terminated_length": 399.0,
|
|
"entropy": 0.9224153310060501,
|
|
"epoch": 0.04365079365079365,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.0008126598149829078,
|
|
"learning_rate": 9.569160997732426e-07,
|
|
"loss": 0.0238,
|
|
"num_tokens": 4613460.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 77,
|
|
"step_time": 49.962997142225504
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1602.0,
|
|
"completions/max_terminated_length": 1602.0,
|
|
"completions/mean_length": 822.953125,
|
|
"completions/mean_terminated_length": 822.953125,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.8187730945646763,
|
|
"epoch": 0.04421768707482993,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.390625,
|
|
"kl": 0.0007310177807084983,
|
|
"learning_rate": 9.563492063492063e-07,
|
|
"loss": 0.0191,
|
|
"num_tokens": 4677641.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 78,
|
|
"step_time": 62.56507764104754
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1495.0,
|
|
"completions/max_terminated_length": 1495.0,
|
|
"completions/mean_length": 792.046875,
|
|
"completions/mean_terminated_length": 792.046875,
|
|
"completions/min_length": 406.0,
|
|
"completions/min_terminated_length": 406.0,
|
|
"entropy": 0.772476501762867,
|
|
"epoch": 0.04478458049886622,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.373046875,
|
|
"kl": 0.0007511713683925336,
|
|
"learning_rate": 9.5578231292517e-07,
|
|
"loss": 0.0328,
|
|
"num_tokens": 4738996.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 79,
|
|
"step_time": 57.763637254945934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1732.0,
|
|
"completions/max_terminated_length": 1732.0,
|
|
"completions/mean_length": 801.203125,
|
|
"completions/mean_terminated_length": 801.203125,
|
|
"completions/min_length": 182.0,
|
|
"completions/min_terminated_length": 182.0,
|
|
"entropy": 0.8073384631425142,
|
|
"epoch": 0.045351473922902494,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.0006839092584414175,
|
|
"learning_rate": 9.552154195011338e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 4800513.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 80,
|
|
"step_time": 67.18388975318521
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1291.0,
|
|
"completions/max_terminated_length": 1291.0,
|
|
"completions/mean_length": 736.703125,
|
|
"completions/mean_terminated_length": 736.703125,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.8549042195081711,
|
|
"epoch": 0.04591836734693878,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.0007158639236877207,
|
|
"learning_rate": 9.546485260770974e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 4857286.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 81,
|
|
"step_time": 50.64162682276219
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1231.0,
|
|
"completions/max_terminated_length": 1231.0,
|
|
"completions/mean_length": 701.421875,
|
|
"completions/mean_terminated_length": 701.421875,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.9150978699326515,
|
|
"epoch": 0.046485260770975055,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0002899169921875,
|
|
"kl": 0.0007863493265176658,
|
|
"learning_rate": 9.54081632653061e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4916737.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 82,
|
|
"step_time": 51.29425719939172
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1905.0,
|
|
"completions/max_terminated_length": 1905.0,
|
|
"completions/mean_length": 811.9375,
|
|
"completions/mean_terminated_length": 811.9375,
|
|
"completions/min_length": 34.0,
|
|
"completions/min_terminated_length": 34.0,
|
|
"entropy": 0.6648439597338438,
|
|
"epoch": 0.04705215419501134,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.0006040604894224089,
|
|
"learning_rate": 9.53514739229025e-07,
|
|
"loss": 0.0176,
|
|
"num_tokens": 4977661.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 83,
|
|
"step_time": 75.30042485240847
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1192.0,
|
|
"completions/max_terminated_length": 1192.0,
|
|
"completions/mean_length": 784.046875,
|
|
"completions/mean_terminated_length": 784.046875,
|
|
"completions/min_length": 382.0,
|
|
"completions/min_terminated_length": 382.0,
|
|
"entropy": 1.0095579735934734,
|
|
"epoch": 0.047619047619047616,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.38671875,
|
|
"kl": 0.000779534788307501,
|
|
"learning_rate": 9.529478458049887e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 5038568.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 84,
|
|
"step_time": 47.187259843572974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1261.0,
|
|
"completions/max_terminated_length": 1261.0,
|
|
"completions/mean_length": 925.625,
|
|
"completions/mean_terminated_length": 925.625,
|
|
"completions/min_length": 563.0,
|
|
"completions/min_terminated_length": 563.0,
|
|
"entropy": 0.6331640388816595,
|
|
"epoch": 0.0481859410430839,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.000579886636842275,
|
|
"learning_rate": 9.523809523809522e-07,
|
|
"loss": -0.0118,
|
|
"num_tokens": 5109528.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 85,
|
|
"step_time": 50.76587761286646
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1555.0,
|
|
"completions/max_terminated_length": 1555.0,
|
|
"completions/mean_length": 810.25,
|
|
"completions/mean_terminated_length": 810.25,
|
|
"completions/min_length": 421.0,
|
|
"completions/min_terminated_length": 421.0,
|
|
"entropy": 0.9099563546478748,
|
|
"epoch": 0.048752834467120185,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.0008195096816052683,
|
|
"learning_rate": 9.51814058956916e-07,
|
|
"loss": -0.0075,
|
|
"num_tokens": 5172280.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 86,
|
|
"step_time": 61.36391172464937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1628.0,
|
|
"completions/max_terminated_length": 1628.0,
|
|
"completions/mean_length": 822.515625,
|
|
"completions/mean_terminated_length": 822.515625,
|
|
"completions/min_length": 401.0,
|
|
"completions/min_terminated_length": 401.0,
|
|
"entropy": 0.9575734511017799,
|
|
"epoch": 0.04931972789115646,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.0007745505245111417,
|
|
"learning_rate": 9.512471655328797e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 5237673.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 87,
|
|
"step_time": 63.86270225327462
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1300.0,
|
|
"completions/max_terminated_length": 1300.0,
|
|
"completions/mean_length": 801.8125,
|
|
"completions/mean_terminated_length": 801.8125,
|
|
"completions/min_length": 445.0,
|
|
"completions/min_terminated_length": 445.0,
|
|
"entropy": 0.9237263891845942,
|
|
"epoch": 0.049886621315192746,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2451171875,
|
|
"kl": 0.0007447822299582185,
|
|
"learning_rate": 9.506802721088435e-07,
|
|
"loss": 0.0031,
|
|
"num_tokens": 5308589.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 88,
|
|
"step_time": 57.35689403023571
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1207.0,
|
|
"completions/max_terminated_length": 1207.0,
|
|
"completions/mean_length": 754.203125,
|
|
"completions/mean_terminated_length": 754.203125,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.8471732772886753,
|
|
"epoch": 0.05045351473922902,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.000293731689453125,
|
|
"kl": 0.0007413623989123153,
|
|
"learning_rate": 9.501133786848072e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 5367594.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 89,
|
|
"step_time": 47.73160739801824
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1388.0,
|
|
"completions/max_terminated_length": 1388.0,
|
|
"completions/mean_length": 787.71875,
|
|
"completions/mean_terminated_length": 787.71875,
|
|
"completions/min_length": 289.0,
|
|
"completions/min_terminated_length": 289.0,
|
|
"entropy": 0.8667267635464668,
|
|
"epoch": 0.05102040816326531,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.000806073730927892,
|
|
"learning_rate": 9.49546485260771e-07,
|
|
"loss": 0.0017,
|
|
"num_tokens": 5429440.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 90,
|
|
"step_time": 54.461288248188794
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1508.0,
|
|
"completions/max_terminated_length": 1508.0,
|
|
"completions/mean_length": 801.96875,
|
|
"completions/mean_terminated_length": 801.96875,
|
|
"completions/min_length": 421.0,
|
|
"completions/min_terminated_length": 421.0,
|
|
"entropy": 0.8943926393985748,
|
|
"epoch": 0.051587301587301584,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.39453125,
|
|
"kl": 0.0007918762676126789,
|
|
"learning_rate": 9.489795918367347e-07,
|
|
"loss": 0.0153,
|
|
"num_tokens": 5493286.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 91,
|
|
"step_time": 59.711739867925644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1538.0,
|
|
"completions/max_terminated_length": 1538.0,
|
|
"completions/mean_length": 811.359375,
|
|
"completions/mean_terminated_length": 811.359375,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.6979294549673796,
|
|
"epoch": 0.05215419501133787,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34765625,
|
|
"kl": 0.000719607574865222,
|
|
"learning_rate": 9.484126984126984e-07,
|
|
"loss": 0.0023,
|
|
"num_tokens": 5555325.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 92,
|
|
"step_time": 59.873963047750294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1534.0,
|
|
"completions/max_terminated_length": 1534.0,
|
|
"completions/mean_length": 763.109375,
|
|
"completions/mean_terminated_length": 763.109375,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 0.933561509475112,
|
|
"epoch": 0.05272108843537415,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.0007509427705372218,
|
|
"learning_rate": 9.47845804988662e-07,
|
|
"loss": 0.0134,
|
|
"num_tokens": 5616124.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 93,
|
|
"step_time": 60.57129462156445
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1352.0,
|
|
"completions/max_terminated_length": 1352.0,
|
|
"completions/mean_length": 685.421875,
|
|
"completions/mean_terminated_length": 685.421875,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 1.1630984470248222,
|
|
"epoch": 0.05328798185941043,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.43359375,
|
|
"kl": 0.0009033987516886555,
|
|
"learning_rate": 9.472789115646258e-07,
|
|
"loss": -0.0213,
|
|
"num_tokens": 5671479.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 94,
|
|
"step_time": 53.713384873233736
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1834.0,
|
|
"completions/max_terminated_length": 1834.0,
|
|
"completions/mean_length": 712.3125,
|
|
"completions/mean_terminated_length": 712.3125,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 0.9153989348560572,
|
|
"epoch": 0.053854875283446714,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.0008535892484360375,
|
|
"learning_rate": 9.467120181405895e-07,
|
|
"loss": -0.0601,
|
|
"num_tokens": 5727011.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 95,
|
|
"step_time": 71.87342350929976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1255.0,
|
|
"completions/max_terminated_length": 1255.0,
|
|
"completions/mean_length": 716.84375,
|
|
"completions/mean_terminated_length": 716.84375,
|
|
"completions/min_length": 82.0,
|
|
"completions/min_terminated_length": 82.0,
|
|
"entropy": 0.7735940665006638,
|
|
"epoch": 0.05442176870748299,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.0007864913895900827,
|
|
"learning_rate": 9.461451247165533e-07,
|
|
"loss": -0.0109,
|
|
"num_tokens": 5783737.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5422958731651306,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 96,
|
|
"step_time": 49.97596748266369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1406.0,
|
|
"completions/max_terminated_length": 1406.0,
|
|
"completions/mean_length": 749.875,
|
|
"completions/mean_terminated_length": 749.875,
|
|
"completions/min_length": 264.0,
|
|
"completions/min_terminated_length": 264.0,
|
|
"entropy": 0.921129334717989,
|
|
"epoch": 0.054988662131519275,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.330078125,
|
|
"kl": 0.0008132914081215858,
|
|
"learning_rate": 9.45578231292517e-07,
|
|
"loss": -0.0044,
|
|
"num_tokens": 5842865.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 97,
|
|
"step_time": 55.25691071804613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1780.0,
|
|
"completions/max_terminated_length": 1780.0,
|
|
"completions/mean_length": 916.84375,
|
|
"completions/mean_terminated_length": 916.84375,
|
|
"completions/min_length": 480.0,
|
|
"completions/min_terminated_length": 480.0,
|
|
"entropy": 0.8317632135003805,
|
|
"epoch": 0.05555555555555555,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.0007306409952434478,
|
|
"learning_rate": 9.450113378684807e-07,
|
|
"loss": 0.0047,
|
|
"num_tokens": 5912511.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 98,
|
|
"step_time": 70.03312500286847
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1460.0,
|
|
"completions/max_terminated_length": 1460.0,
|
|
"completions/mean_length": 824.359375,
|
|
"completions/mean_terminated_length": 824.359375,
|
|
"completions/min_length": 245.0,
|
|
"completions/min_terminated_length": 245.0,
|
|
"entropy": 0.7674974091351032,
|
|
"epoch": 0.05612244897959184,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.185546875,
|
|
"kl": 0.0006871784426039085,
|
|
"learning_rate": 9.444444444444444e-07,
|
|
"loss": 0.0055,
|
|
"num_tokens": 5978486.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 99,
|
|
"step_time": 60.27627642452717
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1437.0,
|
|
"completions/max_terminated_length": 1437.0,
|
|
"completions/mean_length": 852.078125,
|
|
"completions/mean_terminated_length": 852.078125,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.7974452953785658,
|
|
"epoch": 0.05668934240362812,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.0007822227889846545,
|
|
"learning_rate": 9.438775510204082e-07,
|
|
"loss": 0.0245,
|
|
"num_tokens": 6043827.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 100,
|
|
"step_time": 57.44859805330634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1418.0,
|
|
"completions/max_terminated_length": 1418.0,
|
|
"completions/mean_length": 756.625,
|
|
"completions/mean_terminated_length": 756.625,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7994147948920727,
|
|
"epoch": 0.0572562358276644,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.0007528209171141498,
|
|
"learning_rate": 9.433106575963718e-07,
|
|
"loss": 0.026,
|
|
"num_tokens": 6101947.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 101,
|
|
"step_time": 55.89661477226764
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1311.0,
|
|
"completions/max_terminated_length": 1311.0,
|
|
"completions/mean_length": 785.234375,
|
|
"completions/mean_terminated_length": 785.234375,
|
|
"completions/min_length": 382.0,
|
|
"completions/min_terminated_length": 382.0,
|
|
"entropy": 0.6613637581467628,
|
|
"epoch": 0.05782312925170068,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.259765625,
|
|
"kl": 0.0007186354232544545,
|
|
"learning_rate": 9.427437641723356e-07,
|
|
"loss": 0.0114,
|
|
"num_tokens": 6162258.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 102,
|
|
"step_time": 51.73925363551825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1594.0,
|
|
"completions/max_terminated_length": 1594.0,
|
|
"completions/mean_length": 845.875,
|
|
"completions/mean_terminated_length": 845.875,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.8546389937400818,
|
|
"epoch": 0.05839002267573696,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.0007117596451280406,
|
|
"learning_rate": 9.421768707482993e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 6226426.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 103,
|
|
"step_time": 62.46534308884293
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1694.0,
|
|
"completions/max_terminated_length": 1694.0,
|
|
"completions/mean_length": 844.265625,
|
|
"completions/mean_terminated_length": 844.265625,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.6533396132290363,
|
|
"epoch": 0.05895691609977324,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.0008017468153411755,
|
|
"learning_rate": 9.41609977324263e-07,
|
|
"loss": 0.02,
|
|
"num_tokens": 6293019.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 104,
|
|
"step_time": 66.67652920912951
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/max_terminated_length": 1389.0,
|
|
"completions/mean_length": 734.3125,
|
|
"completions/mean_terminated_length": 734.3125,
|
|
"completions/min_length": 253.0,
|
|
"completions/min_terminated_length": 253.0,
|
|
"entropy": 0.8464022967964411,
|
|
"epoch": 0.05952380952380952,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.0008064001049206126,
|
|
"learning_rate": 9.410430839002267e-07,
|
|
"loss": 0.0259,
|
|
"num_tokens": 6349567.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 105,
|
|
"step_time": 54.00935931224376
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1550.0,
|
|
"completions/max_terminated_length": 1550.0,
|
|
"completions/mean_length": 770.546875,
|
|
"completions/mean_terminated_length": 770.546875,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.8186229839920998,
|
|
"epoch": 0.060090702947845805,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.0007566980402771151,
|
|
"learning_rate": 9.404761904761904e-07,
|
|
"loss": -0.0066,
|
|
"num_tokens": 6408538.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5883605480194092,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 106,
|
|
"step_time": 60.694702457636595
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1288.0,
|
|
"completions/max_terminated_length": 1288.0,
|
|
"completions/mean_length": 758.34375,
|
|
"completions/mean_terminated_length": 758.34375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 0.8053422253578901,
|
|
"epoch": 0.06065759637188209,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.0007592442962049972,
|
|
"learning_rate": 9.399092970521542e-07,
|
|
"loss": 0.0093,
|
|
"num_tokens": 6469112.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 107,
|
|
"step_time": 50.79260412603617
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1458.0,
|
|
"completions/max_terminated_length": 1458.0,
|
|
"completions/mean_length": 797.53125,
|
|
"completions/mean_terminated_length": 797.53125,
|
|
"completions/min_length": 352.0,
|
|
"completions/min_terminated_length": 352.0,
|
|
"entropy": 0.6849835515022278,
|
|
"epoch": 0.061224489795918366,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.0007152012603910407,
|
|
"learning_rate": 9.393424036281179e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 6529466.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 108,
|
|
"step_time": 57.19498935434967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1308.0,
|
|
"completions/max_terminated_length": 1308.0,
|
|
"completions/mean_length": 659.4375,
|
|
"completions/mean_terminated_length": 659.4375,
|
|
"completions/min_length": 215.0,
|
|
"completions/min_terminated_length": 215.0,
|
|
"entropy": 1.0229039713740349,
|
|
"epoch": 0.06179138321995465,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.384765625,
|
|
"kl": 0.0009435774336452596,
|
|
"learning_rate": 9.387755102040816e-07,
|
|
"loss": 0.0059,
|
|
"num_tokens": 6580846.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 109,
|
|
"step_time": 51.30871072318405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1400.0,
|
|
"completions/max_terminated_length": 1400.0,
|
|
"completions/mean_length": 726.390625,
|
|
"completions/mean_terminated_length": 726.390625,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.7591048255562782,
|
|
"epoch": 0.06235827664399093,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.0007802735126460902,
|
|
"learning_rate": 9.382086167800452e-07,
|
|
"loss": -0.0063,
|
|
"num_tokens": 6637271.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 110,
|
|
"step_time": 55.055217486806214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1309.0,
|
|
"completions/max_terminated_length": 1309.0,
|
|
"completions/mean_length": 742.71875,
|
|
"completions/mean_terminated_length": 742.71875,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.9038644768297672,
|
|
"epoch": 0.06292517006802721,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.0007633761742908973,
|
|
"learning_rate": 9.37641723356009e-07,
|
|
"loss": 0.0132,
|
|
"num_tokens": 6696757.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 111,
|
|
"step_time": 51.84569195844233
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1390.0,
|
|
"completions/max_terminated_length": 1390.0,
|
|
"completions/mean_length": 762.453125,
|
|
"completions/mean_terminated_length": 762.453125,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.7126942295581102,
|
|
"epoch": 0.06349206349206349,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.000408172607421875,
|
|
"kl": 0.0008155866780725773,
|
|
"learning_rate": 9.370748299319727e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 6755994.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 112,
|
|
"step_time": 54.81356189586222
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1438.0,
|
|
"completions/max_terminated_length": 1438.0,
|
|
"completions/mean_length": 777.125,
|
|
"completions/mean_terminated_length": 777.125,
|
|
"completions/min_length": 295.0,
|
|
"completions/min_terminated_length": 295.0,
|
|
"entropy": 0.8607089705765247,
|
|
"epoch": 0.06405895691609978,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.0008256015426013619,
|
|
"learning_rate": 9.365079365079365e-07,
|
|
"loss": -0.0086,
|
|
"num_tokens": 6817346.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 113,
|
|
"step_time": 57.42120492178947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 744.3125,
|
|
"completions/mean_terminated_length": 744.3125,
|
|
"completions/min_length": 222.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"entropy": 0.9622498042881489,
|
|
"epoch": 0.06462585034013606,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.333984375,
|
|
"kl": 0.0008918357489164919,
|
|
"learning_rate": 9.359410430839002e-07,
|
|
"loss": 0.015,
|
|
"num_tokens": 6877462.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 114,
|
|
"step_time": 54.03743090759963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1034.0,
|
|
"completions/max_terminated_length": 1034.0,
|
|
"completions/mean_length": 652.59375,
|
|
"completions/mean_terminated_length": 652.59375,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 1.0135034248232841,
|
|
"epoch": 0.06519274376417233,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.69140625,
|
|
"kl": 0.0010756870760815218,
|
|
"learning_rate": 9.35374149659864e-07,
|
|
"loss": -0.0047,
|
|
"num_tokens": 6929452.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 115,
|
|
"step_time": 41.20539268106222
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1209.0,
|
|
"completions/max_terminated_length": 1209.0,
|
|
"completions/mean_length": 697.765625,
|
|
"completions/mean_terminated_length": 697.765625,
|
|
"completions/min_length": 325.0,
|
|
"completions/min_terminated_length": 325.0,
|
|
"entropy": 0.8065849877893925,
|
|
"epoch": 0.06575963718820861,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6640625,
|
|
"kl": 0.0009536606376059353,
|
|
"learning_rate": 9.348072562358276e-07,
|
|
"loss": -0.0416,
|
|
"num_tokens": 6990093.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 116,
|
|
"step_time": 51.148926474153996
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1358.0,
|
|
"completions/max_terminated_length": 1358.0,
|
|
"completions/mean_length": 774.703125,
|
|
"completions/mean_terminated_length": 774.703125,
|
|
"completions/min_length": 364.0,
|
|
"completions/min_terminated_length": 364.0,
|
|
"entropy": 0.8654811605811119,
|
|
"epoch": 0.0663265306122449,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.0008410576774622314,
|
|
"learning_rate": 9.342403628117913e-07,
|
|
"loss": 0.0157,
|
|
"num_tokens": 7053066.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 117,
|
|
"step_time": 55.61251427512616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1278.0,
|
|
"completions/max_terminated_length": 1278.0,
|
|
"completions/mean_length": 691.921875,
|
|
"completions/mean_terminated_length": 691.921875,
|
|
"completions/min_length": 322.0,
|
|
"completions/min_terminated_length": 322.0,
|
|
"entropy": 0.9328459855169058,
|
|
"epoch": 0.06689342403628118,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.0009202329711115453,
|
|
"learning_rate": 9.33673469387755e-07,
|
|
"loss": 0.0169,
|
|
"num_tokens": 7109197.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 118,
|
|
"step_time": 51.24461770057678
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1603.0,
|
|
"completions/max_terminated_length": 1603.0,
|
|
"completions/mean_length": 726.078125,
|
|
"completions/mean_terminated_length": 726.078125,
|
|
"completions/min_length": 259.0,
|
|
"completions/min_terminated_length": 259.0,
|
|
"entropy": 0.8497186675667763,
|
|
"epoch": 0.06746031746031746,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4765625,
|
|
"kl": 0.0008916753904486541,
|
|
"learning_rate": 9.331065759637188e-07,
|
|
"loss": -0.0135,
|
|
"num_tokens": 7164370.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 119,
|
|
"step_time": 62.421043317765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1325.0,
|
|
"completions/max_terminated_length": 1325.0,
|
|
"completions/mean_length": 864.21875,
|
|
"completions/mean_terminated_length": 864.21875,
|
|
"completions/min_length": 526.0,
|
|
"completions/min_terminated_length": 526.0,
|
|
"entropy": 0.8159789443016052,
|
|
"epoch": 0.06802721088435375,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.0007719660407019546,
|
|
"learning_rate": 9.325396825396825e-07,
|
|
"loss": 0.0131,
|
|
"num_tokens": 7230192.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.666620135307312,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 120,
|
|
"step_time": 51.65067602414638
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2271.0,
|
|
"completions/max_terminated_length": 2271.0,
|
|
"completions/mean_length": 818.578125,
|
|
"completions/mean_terminated_length": 818.578125,
|
|
"completions/min_length": 387.0,
|
|
"completions/min_terminated_length": 387.0,
|
|
"entropy": 0.6372475307434797,
|
|
"epoch": 0.06859410430839002,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.0007827264707884751,
|
|
"learning_rate": 9.319727891156463e-07,
|
|
"loss": 0.0045,
|
|
"num_tokens": 7294253.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 121,
|
|
"step_time": 89.85847285855561
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1541.0,
|
|
"completions/max_terminated_length": 1541.0,
|
|
"completions/mean_length": 827.15625,
|
|
"completions/mean_terminated_length": 827.15625,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 0.666001146659255,
|
|
"epoch": 0.0691609977324263,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.498046875,
|
|
"kl": 0.0007001044541539159,
|
|
"learning_rate": 9.3140589569161e-07,
|
|
"loss": 0.0055,
|
|
"num_tokens": 7358327.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7007648944854736,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 122,
|
|
"step_time": 60.366386415436864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1220.0,
|
|
"completions/max_terminated_length": 1220.0,
|
|
"completions/mean_length": 598.796875,
|
|
"completions/mean_terminated_length": 598.796875,
|
|
"completions/min_length": 195.0,
|
|
"completions/min_terminated_length": 195.0,
|
|
"entropy": 1.0049929097294807,
|
|
"epoch": 0.06972789115646258,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.30078125,
|
|
"kl": 0.0010934763122349977,
|
|
"learning_rate": 9.308390022675737e-07,
|
|
"loss": -0.0009,
|
|
"num_tokens": 7410298.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 123,
|
|
"step_time": 48.925475688651204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1826.0,
|
|
"completions/max_terminated_length": 1826.0,
|
|
"completions/mean_length": 602.09375,
|
|
"completions/mean_terminated_length": 602.09375,
|
|
"completions/min_length": 231.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"entropy": 0.9215579964220524,
|
|
"epoch": 0.07029478458049887,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.69140625,
|
|
"kl": 0.0012636464562092442,
|
|
"learning_rate": 9.302721088435374e-07,
|
|
"loss": -0.0067,
|
|
"num_tokens": 7458544.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 124,
|
|
"step_time": 70.9872962711379
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1610.0,
|
|
"completions/max_terminated_length": 1610.0,
|
|
"completions/mean_length": 830.53125,
|
|
"completions/mean_terminated_length": 830.53125,
|
|
"completions/min_length": 351.0,
|
|
"completions/min_terminated_length": 351.0,
|
|
"entropy": 0.8522497117519379,
|
|
"epoch": 0.07086167800453515,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.0009019470344355796,
|
|
"learning_rate": 9.29705215419501e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 7524058.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 125,
|
|
"step_time": 63.3850915748626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1282.0,
|
|
"completions/max_terminated_length": 1282.0,
|
|
"completions/mean_length": 806.25,
|
|
"completions/mean_terminated_length": 806.25,
|
|
"completions/min_length": 246.0,
|
|
"completions/min_terminated_length": 246.0,
|
|
"entropy": 0.7999003082513809,
|
|
"epoch": 0.07142857142857142,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.0008742077352508204,
|
|
"learning_rate": 9.291383219954648e-07,
|
|
"loss": -0.0328,
|
|
"num_tokens": 7586338.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 126,
|
|
"step_time": 50.78821122366935
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1572.0,
|
|
"completions/max_terminated_length": 1572.0,
|
|
"completions/mean_length": 790.1875,
|
|
"completions/mean_terminated_length": 790.1875,
|
|
"completions/min_length": 421.0,
|
|
"completions/min_terminated_length": 421.0,
|
|
"entropy": 0.8215701729059219,
|
|
"epoch": 0.07199546485260772,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.0009467004892940167,
|
|
"learning_rate": 9.285714285714285e-07,
|
|
"loss": 0.0209,
|
|
"num_tokens": 7648062.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 127,
|
|
"step_time": 61.55685986764729
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1501.0,
|
|
"completions/max_terminated_length": 1501.0,
|
|
"completions/mean_length": 718.5,
|
|
"completions/mean_terminated_length": 718.5,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.8467830922454596,
|
|
"epoch": 0.07256235827664399,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2890625,
|
|
"kl": 0.0010289369311067276,
|
|
"learning_rate": 9.280045351473923e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 7704590.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 128,
|
|
"step_time": 58.87193254288286
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1444.0,
|
|
"completions/max_terminated_length": 1444.0,
|
|
"completions/mean_length": 789.75,
|
|
"completions/mean_terminated_length": 789.75,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.6771079618483782,
|
|
"epoch": 0.07312925170068027,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.0008786784237599932,
|
|
"learning_rate": 9.274376417233559e-07,
|
|
"loss": 0.0014,
|
|
"num_tokens": 7764822.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 129,
|
|
"step_time": 56.4890464367345
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2020.0,
|
|
"completions/mean_length": 925.453125,
|
|
"completions/mean_terminated_length": 875.1270141601562,
|
|
"completions/min_length": 389.0,
|
|
"completions/min_terminated_length": 389.0,
|
|
"entropy": 0.6869203355163336,
|
|
"epoch": 0.07369614512471655,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.0009084341945708729,
|
|
"learning_rate": 9.268707482993197e-07,
|
|
"loss": -0.0242,
|
|
"num_tokens": 7834419.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 130,
|
|
"step_time": 181.55315329879522
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1208.0,
|
|
"completions/max_terminated_length": 1208.0,
|
|
"completions/mean_length": 697.703125,
|
|
"completions/mean_terminated_length": 697.703125,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 1.0916701704263687,
|
|
"epoch": 0.07426303854875284,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.32421875,
|
|
"kl": 0.0010625670292938594,
|
|
"learning_rate": 9.263038548752834e-07,
|
|
"loss": -0.0137,
|
|
"num_tokens": 7889696.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 131,
|
|
"step_time": 47.96899611968547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1471.0,
|
|
"completions/max_terminated_length": 1471.0,
|
|
"completions/mean_length": 795.75,
|
|
"completions/mean_terminated_length": 795.75,
|
|
"completions/min_length": 293.0,
|
|
"completions/min_terminated_length": 293.0,
|
|
"entropy": 0.7799660600721836,
|
|
"epoch": 0.07482993197278912,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3828125,
|
|
"kl": 0.0009379767434438691,
|
|
"learning_rate": 9.257369614512472e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 7951104.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 132,
|
|
"step_time": 58.739000890403986
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1454.0,
|
|
"completions/max_terminated_length": 1454.0,
|
|
"completions/mean_length": 665.484375,
|
|
"completions/mean_terminated_length": 665.484375,
|
|
"completions/min_length": 240.0,
|
|
"completions/min_terminated_length": 240.0,
|
|
"entropy": 0.8104948624968529,
|
|
"epoch": 0.07539682539682539,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.001190572242194321,
|
|
"learning_rate": 9.251700680272108e-07,
|
|
"loss": 0.0302,
|
|
"num_tokens": 8004183.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 133,
|
|
"step_time": 57.126070658676326
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1956.0,
|
|
"completions/max_terminated_length": 1956.0,
|
|
"completions/mean_length": 827.75,
|
|
"completions/mean_terminated_length": 827.75,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.7314121834933758,
|
|
"epoch": 0.07596371882086168,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.000903994710824918,
|
|
"learning_rate": 9.246031746031746e-07,
|
|
"loss": -0.0049,
|
|
"num_tokens": 8068271.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5883605480194092,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 134,
|
|
"step_time": 77.03828375507146
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1391.0,
|
|
"completions/max_terminated_length": 1391.0,
|
|
"completions/mean_length": 697.5,
|
|
"completions/mean_terminated_length": 697.5,
|
|
"completions/min_length": 307.0,
|
|
"completions/min_terminated_length": 307.0,
|
|
"entropy": 0.7819268852472305,
|
|
"epoch": 0.07653061224489796,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.0012548193262773566,
|
|
"learning_rate": 9.240362811791382e-07,
|
|
"loss": -0.0229,
|
|
"num_tokens": 8122103.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.71200031042099,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 135,
|
|
"step_time": 53.75756964739412
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1664.0,
|
|
"completions/max_terminated_length": 1664.0,
|
|
"completions/mean_length": 855.53125,
|
|
"completions/mean_terminated_length": 855.53125,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.7436029799282551,
|
|
"epoch": 0.07709750566893424,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.001049405378580559,
|
|
"learning_rate": 9.23469387755102e-07,
|
|
"loss": 0.0069,
|
|
"num_tokens": 8186665.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 136,
|
|
"step_time": 64.77417926304042
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1187.0,
|
|
"completions/max_terminated_length": 1187.0,
|
|
"completions/mean_length": 681.0625,
|
|
"completions/mean_terminated_length": 681.0625,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.9490941390395164,
|
|
"epoch": 0.07766439909297052,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.0011216318016522564,
|
|
"learning_rate": 9.229024943310657e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 8240965.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 137,
|
|
"step_time": 47.23937799781561
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1421.0,
|
|
"completions/max_terminated_length": 1421.0,
|
|
"completions/mean_length": 748.65625,
|
|
"completions/mean_terminated_length": 748.65625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.9417592454701662,
|
|
"epoch": 0.0782312925170068,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.369140625,
|
|
"kl": 0.0010869996040128171,
|
|
"learning_rate": 9.223356009070295e-07,
|
|
"loss": -0.0135,
|
|
"num_tokens": 8302119.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 138,
|
|
"step_time": 56.91432747989893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1174.0,
|
|
"completions/max_terminated_length": 1174.0,
|
|
"completions/mean_length": 727.1875,
|
|
"completions/mean_terminated_length": 727.1875,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.8250185586512089,
|
|
"epoch": 0.07879818594104308,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2275390625,
|
|
"kl": 0.0011217291685170494,
|
|
"learning_rate": 9.217687074829932e-07,
|
|
"loss": 0.0089,
|
|
"num_tokens": 8360459.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 139,
|
|
"step_time": 46.688301916234195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1552.0,
|
|
"completions/max_terminated_length": 1552.0,
|
|
"completions/mean_length": 830.15625,
|
|
"completions/mean_terminated_length": 830.15625,
|
|
"completions/min_length": 378.0,
|
|
"completions/min_terminated_length": 378.0,
|
|
"entropy": 0.6795956548303366,
|
|
"epoch": 0.07936507936507936,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.39453125,
|
|
"kl": 0.0011381846597942058,
|
|
"learning_rate": 9.21201814058957e-07,
|
|
"loss": -0.0006,
|
|
"num_tokens": 8422341.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 140,
|
|
"step_time": 60.8731788918376
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1632.0,
|
|
"completions/max_terminated_length": 1632.0,
|
|
"completions/mean_length": 707.375,
|
|
"completions/mean_terminated_length": 707.375,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.8753399010747671,
|
|
"epoch": 0.07993197278911565,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.0014630026489612646,
|
|
"learning_rate": 9.206349206349205e-07,
|
|
"loss": 0.0028,
|
|
"num_tokens": 8477165.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 141,
|
|
"step_time": 64.22281729802489
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1319.0,
|
|
"completions/max_terminated_length": 1319.0,
|
|
"completions/mean_length": 809.390625,
|
|
"completions/mean_terminated_length": 809.390625,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 0.9984244965016842,
|
|
"epoch": 0.08049886621315193,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2021484375,
|
|
"kl": 0.0011269914175500162,
|
|
"learning_rate": 9.200680272108843e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 8539014.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 142,
|
|
"step_time": 51.55915431212634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1253.0,
|
|
"completions/max_terminated_length": 1253.0,
|
|
"completions/mean_length": 814.359375,
|
|
"completions/mean_terminated_length": 814.359375,
|
|
"completions/min_length": 466.0,
|
|
"completions/min_terminated_length": 466.0,
|
|
"entropy": 0.9719417598098516,
|
|
"epoch": 0.0810657596371882,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4765625,
|
|
"kl": 0.0009653525630710647,
|
|
"learning_rate": 9.19501133786848e-07,
|
|
"loss": 0.0104,
|
|
"num_tokens": 8601453.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5422958731651306,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 143,
|
|
"step_time": 50.030846861191094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1326.0,
|
|
"completions/max_terminated_length": 1326.0,
|
|
"completions/mean_length": 650.203125,
|
|
"completions/mean_terminated_length": 650.203125,
|
|
"completions/min_length": 47.0,
|
|
"completions/min_terminated_length": 47.0,
|
|
"entropy": 0.9308004006743431,
|
|
"epoch": 0.08163265306122448,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4375,
|
|
"kl": 0.002023561413807329,
|
|
"learning_rate": 9.189342403628118e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 8653642.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 144,
|
|
"step_time": 51.93653193488717
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 787.859375,
|
|
"completions/mean_terminated_length": 787.859375,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.8656461779028177,
|
|
"epoch": 0.08219954648526077,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.390625,
|
|
"kl": 0.0011374048954166938,
|
|
"learning_rate": 9.183673469387755e-07,
|
|
"loss": -0.0089,
|
|
"num_tokens": 8715849.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 145,
|
|
"step_time": 51.37489194236696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1240.0,
|
|
"completions/mean_length": 791.65625,
|
|
"completions/mean_terminated_length": 739.2064208984375,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.8582950960844755,
|
|
"epoch": 0.08276643990929705,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.0014971747732488438,
|
|
"learning_rate": 9.178004535147393e-07,
|
|
"loss": -0.0031,
|
|
"num_tokens": 8779851.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 146,
|
|
"step_time": 184.58719532191753
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1267.0,
|
|
"completions/max_terminated_length": 1267.0,
|
|
"completions/mean_length": 861.484375,
|
|
"completions/mean_terminated_length": 861.484375,
|
|
"completions/min_length": 475.0,
|
|
"completions/min_terminated_length": 475.0,
|
|
"entropy": 0.8132954798638821,
|
|
"epoch": 0.08333333333333333,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.001158217251941096,
|
|
"learning_rate": 9.172335600907029e-07,
|
|
"loss": 0.0055,
|
|
"num_tokens": 8845346.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 147,
|
|
"step_time": 49.048096066340804
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1306.0,
|
|
"completions/max_terminated_length": 1306.0,
|
|
"completions/mean_length": 756.921875,
|
|
"completions/mean_terminated_length": 756.921875,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.8934296295046806,
|
|
"epoch": 0.08390022675736962,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.400390625,
|
|
"kl": 0.0012354206082818564,
|
|
"learning_rate": 9.166666666666665e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 8903637.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 148,
|
|
"step_time": 50.68960814829916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1395.0,
|
|
"completions/max_terminated_length": 1395.0,
|
|
"completions/mean_length": 807.578125,
|
|
"completions/mean_terminated_length": 807.578125,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.740367965772748,
|
|
"epoch": 0.0844671201814059,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30859375,
|
|
"kl": 0.0012059269793098792,
|
|
"learning_rate": 9.160997732426303e-07,
|
|
"loss": 0.0085,
|
|
"num_tokens": 8965810.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 149,
|
|
"step_time": 54.18200596701354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1610.0,
|
|
"completions/max_terminated_length": 1610.0,
|
|
"completions/mean_length": 812.3125,
|
|
"completions/mean_terminated_length": 812.3125,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.5668262634426355,
|
|
"epoch": 0.08503401360544217,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.0013014152864343487,
|
|
"learning_rate": 9.15532879818594e-07,
|
|
"loss": 0.0183,
|
|
"num_tokens": 9027926.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.69987952709198,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 150,
|
|
"step_time": 62.40979080926627
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1775.0,
|
|
"completions/max_terminated_length": 1775.0,
|
|
"completions/mean_length": 835.859375,
|
|
"completions/mean_terminated_length": 835.859375,
|
|
"completions/min_length": 210.0,
|
|
"completions/min_terminated_length": 210.0,
|
|
"entropy": 0.9155545383691788,
|
|
"epoch": 0.08560090702947845,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.384765625,
|
|
"kl": 0.0012780445722455624,
|
|
"learning_rate": 9.149659863945578e-07,
|
|
"loss": 0.0195,
|
|
"num_tokens": 9091717.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 151,
|
|
"step_time": 69.37227410264313
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1328.0,
|
|
"completions/max_terminated_length": 1328.0,
|
|
"completions/mean_length": 765.390625,
|
|
"completions/mean_terminated_length": 765.390625,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.8690036237239838,
|
|
"epoch": 0.08616780045351474,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.001788135792594403,
|
|
"learning_rate": 9.143990929705215e-07,
|
|
"loss": -0.0196,
|
|
"num_tokens": 9151678.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 152,
|
|
"step_time": 52.06068628747016
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1262.0,
|
|
"completions/mean_length": 783.9375,
|
|
"completions/mean_terminated_length": 731.3651123046875,
|
|
"completions/min_length": 306.0,
|
|
"completions/min_terminated_length": 306.0,
|
|
"entropy": 0.9219446405768394,
|
|
"epoch": 0.08673469387755102,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.0015317276192945428,
|
|
"learning_rate": 9.138321995464853e-07,
|
|
"loss": -0.0009,
|
|
"num_tokens": 9212578.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 153,
|
|
"step_time": 180.3162548020482
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 828.484375,
|
|
"completions/mean_terminated_length": 828.484375,
|
|
"completions/min_length": 484.0,
|
|
"completions/min_terminated_length": 484.0,
|
|
"entropy": 0.867816049605608,
|
|
"epoch": 0.0873015873015873,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.0013349929031392094,
|
|
"learning_rate": 9.132653061224489e-07,
|
|
"loss": 0.0106,
|
|
"num_tokens": 9275665.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 154,
|
|
"step_time": 50.62536738999188
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1414.0,
|
|
"completions/max_terminated_length": 1414.0,
|
|
"completions/mean_length": 749.203125,
|
|
"completions/mean_terminated_length": 749.203125,
|
|
"completions/min_length": 325.0,
|
|
"completions/min_terminated_length": 325.0,
|
|
"entropy": 0.8404502682387829,
|
|
"epoch": 0.08786848072562359,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.0014105687332630623,
|
|
"learning_rate": 9.126984126984127e-07,
|
|
"loss": 0.0045,
|
|
"num_tokens": 9338358.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 155,
|
|
"step_time": 56.59618319571018
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1363.0,
|
|
"completions/max_terminated_length": 1363.0,
|
|
"completions/mean_length": 815.71875,
|
|
"completions/mean_terminated_length": 815.71875,
|
|
"completions/min_length": 410.0,
|
|
"completions/min_terminated_length": 410.0,
|
|
"entropy": 0.7059785295277834,
|
|
"epoch": 0.08843537414965986,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.0014341858404804952,
|
|
"learning_rate": 9.121315192743763e-07,
|
|
"loss": -0.0028,
|
|
"num_tokens": 9401748.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 156,
|
|
"step_time": 53.83780967630446
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1761.0,
|
|
"completions/max_terminated_length": 1761.0,
|
|
"completions/mean_length": 754.5625,
|
|
"completions/mean_terminated_length": 754.5625,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 0.9523781202733517,
|
|
"epoch": 0.08900226757369614,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.0024719537650526036,
|
|
"learning_rate": 9.115646258503401e-07,
|
|
"loss": -0.0176,
|
|
"num_tokens": 9460232.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6878606677055359,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 157,
|
|
"step_time": 69.29571347311139
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1437.0,
|
|
"completions/max_terminated_length": 1437.0,
|
|
"completions/mean_length": 736.546875,
|
|
"completions/mean_terminated_length": 736.546875,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.8924895077943802,
|
|
"epoch": 0.08956916099773243,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.0014391718214028515,
|
|
"learning_rate": 9.109977324263038e-07,
|
|
"loss": -0.0143,
|
|
"num_tokens": 9516203.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 158,
|
|
"step_time": 56.50125625450164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2517.0,
|
|
"completions/max_terminated_length": 2517.0,
|
|
"completions/mean_length": 848.515625,
|
|
"completions/mean_terminated_length": 848.515625,
|
|
"completions/min_length": 340.0,
|
|
"completions/min_terminated_length": 340.0,
|
|
"entropy": 0.7691221535205841,
|
|
"epoch": 0.09013605442176871,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.0012151955052104313,
|
|
"learning_rate": 9.104308390022676e-07,
|
|
"loss": 0.0268,
|
|
"num_tokens": 9581780.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 159,
|
|
"step_time": 100.30379501078278
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3178.0,
|
|
"completions/max_terminated_length": 3178.0,
|
|
"completions/mean_length": 747.359375,
|
|
"completions/mean_terminated_length": 747.359375,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.7731830514967442,
|
|
"epoch": 0.09070294784580499,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.287109375,
|
|
"kl": 0.0014131366970104864,
|
|
"learning_rate": 9.098639455782312e-07,
|
|
"loss": -0.0093,
|
|
"num_tokens": 9638755.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 160,
|
|
"step_time": 129.41354488953948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1599.0,
|
|
"completions/max_terminated_length": 1599.0,
|
|
"completions/mean_length": 847.859375,
|
|
"completions/mean_terminated_length": 847.859375,
|
|
"completions/min_length": 485.0,
|
|
"completions/min_terminated_length": 485.0,
|
|
"entropy": 0.879808509722352,
|
|
"epoch": 0.09126984126984126,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.0015618964862369467,
|
|
"learning_rate": 9.09297052154195e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 9704434.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 161,
|
|
"step_time": 61.810291890986264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1938.0,
|
|
"completions/max_terminated_length": 1938.0,
|
|
"completions/mean_length": 793.609375,
|
|
"completions/mean_terminated_length": 793.609375,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.5231372248381376,
|
|
"epoch": 0.09183673469387756,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.0012791927074431442,
|
|
"learning_rate": 9.087301587301587e-07,
|
|
"loss": 0.052,
|
|
"num_tokens": 9764505.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 162,
|
|
"step_time": 75.29955735895783
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1246.0,
|
|
"completions/max_terminated_length": 1246.0,
|
|
"completions/mean_length": 740.59375,
|
|
"completions/mean_terminated_length": 740.59375,
|
|
"completions/min_length": 410.0,
|
|
"completions/min_terminated_length": 410.0,
|
|
"entropy": 0.957040335983038,
|
|
"epoch": 0.09240362811791383,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4765625,
|
|
"kl": 0.0019501148199196905,
|
|
"learning_rate": 9.081632653061225e-07,
|
|
"loss": 0.0302,
|
|
"num_tokens": 9823023.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 163,
|
|
"step_time": 48.82911225967109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1744.0,
|
|
"completions/max_terminated_length": 1744.0,
|
|
"completions/mean_length": 821.328125,
|
|
"completions/mean_terminated_length": 821.328125,
|
|
"completions/min_length": 464.0,
|
|
"completions/min_terminated_length": 464.0,
|
|
"entropy": 0.9202660918235779,
|
|
"epoch": 0.09297052154195011,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"kl": 0.0016514871385879815,
|
|
"learning_rate": 9.075963718820861e-07,
|
|
"loss": -0.0187,
|
|
"num_tokens": 9887660.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 164,
|
|
"step_time": 67.61680617462844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1266.0,
|
|
"completions/max_terminated_length": 1266.0,
|
|
"completions/mean_length": 723.953125,
|
|
"completions/mean_terminated_length": 723.953125,
|
|
"completions/min_length": 184.0,
|
|
"completions/min_terminated_length": 184.0,
|
|
"entropy": 0.6882897149771452,
|
|
"epoch": 0.0935374149659864,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.396484375,
|
|
"kl": 0.0016417923288827296,
|
|
"learning_rate": 9.070294784580499e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 9942961.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 165,
|
|
"step_time": 49.54639142379165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1351.0,
|
|
"completions/max_terminated_length": 1351.0,
|
|
"completions/mean_length": 721.015625,
|
|
"completions/mean_terminated_length": 721.015625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.8900751657783985,
|
|
"epoch": 0.09410430839002268,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.291015625,
|
|
"kl": 0.0020163542503723875,
|
|
"learning_rate": 9.064625850340135e-07,
|
|
"loss": -0.0039,
|
|
"num_tokens": 10000034.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 166,
|
|
"step_time": 54.27309240680188
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1461.0,
|
|
"completions/max_terminated_length": 1461.0,
|
|
"completions/mean_length": 768.78125,
|
|
"completions/mean_terminated_length": 768.78125,
|
|
"completions/min_length": 249.0,
|
|
"completions/min_terminated_length": 249.0,
|
|
"entropy": 0.6647769100964069,
|
|
"epoch": 0.09467120181405896,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.001808562345104292,
|
|
"learning_rate": 9.058956916099772e-07,
|
|
"loss": 0.0153,
|
|
"num_tokens": 10059292.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.7331100702285767,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 167,
|
|
"step_time": 57.41250906791538
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1128.0,
|
|
"completions/max_terminated_length": 1128.0,
|
|
"completions/mean_length": 678.5625,
|
|
"completions/mean_terminated_length": 678.5625,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.8922697510570288,
|
|
"epoch": 0.09523809523809523,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.0014247586659621447,
|
|
"learning_rate": 9.05328798185941e-07,
|
|
"loss": 0.0131,
|
|
"num_tokens": 10113944.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 168,
|
|
"step_time": 45.19113690778613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 818.234375,
|
|
"completions/mean_terminated_length": 818.234375,
|
|
"completions/min_length": 481.0,
|
|
"completions/min_terminated_length": 481.0,
|
|
"entropy": 0.9066624715924263,
|
|
"epoch": 0.09580498866213152,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.0017615591932553798,
|
|
"learning_rate": 9.047619047619047e-07,
|
|
"loss": -0.0157,
|
|
"num_tokens": 10177631.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 169,
|
|
"step_time": 53.81898599676788
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1307.0,
|
|
"completions/max_terminated_length": 1307.0,
|
|
"completions/mean_length": 701.015625,
|
|
"completions/mean_terminated_length": 701.015625,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.9111842215061188,
|
|
"epoch": 0.0963718820861678,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.447265625,
|
|
"kl": 0.0025995390242314897,
|
|
"learning_rate": 9.041950113378685e-07,
|
|
"loss": 0.0198,
|
|
"num_tokens": 10236248.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 170,
|
|
"step_time": 52.9567281100899
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1325.0,
|
|
"completions/max_terminated_length": 1325.0,
|
|
"completions/mean_length": 701.6875,
|
|
"completions/mean_terminated_length": 701.6875,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.8338793143630028,
|
|
"epoch": 0.09693877551020408,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.0017993416768149473,
|
|
"learning_rate": 9.036281179138322e-07,
|
|
"loss": -0.0165,
|
|
"num_tokens": 10290100.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 171,
|
|
"step_time": 52.467592406086624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1762.0,
|
|
"completions/max_terminated_length": 1762.0,
|
|
"completions/mean_length": 759.390625,
|
|
"completions/mean_terminated_length": 759.390625,
|
|
"completions/min_length": 333.0,
|
|
"completions/min_terminated_length": 333.0,
|
|
"entropy": 0.9258961863815784,
|
|
"epoch": 0.09750566893424037,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.263671875,
|
|
"kl": 0.002015009449678473,
|
|
"learning_rate": 9.030612244897958e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 10348733.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 172,
|
|
"step_time": 68.8363786675036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1413.0,
|
|
"completions/max_terminated_length": 1413.0,
|
|
"completions/mean_length": 765.84375,
|
|
"completions/mean_terminated_length": 765.84375,
|
|
"completions/min_length": 407.0,
|
|
"completions/min_terminated_length": 407.0,
|
|
"entropy": 0.9442645497620106,
|
|
"epoch": 0.09807256235827665,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.0017031964925990906,
|
|
"learning_rate": 9.024943310657595e-07,
|
|
"loss": 0.011,
|
|
"num_tokens": 10407603.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 173,
|
|
"step_time": 54.379828153178096
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1700.0,
|
|
"completions/max_terminated_length": 1700.0,
|
|
"completions/mean_length": 864.640625,
|
|
"completions/mean_terminated_length": 864.640625,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.9839955344796181,
|
|
"epoch": 0.09863945578231292,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.234375,
|
|
"kl": 0.0015942358804750256,
|
|
"learning_rate": 9.019274376417233e-07,
|
|
"loss": 0.0186,
|
|
"num_tokens": 10473524.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 174,
|
|
"step_time": 66.9548751115799
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1157.0,
|
|
"completions/max_terminated_length": 1157.0,
|
|
"completions/mean_length": 778.703125,
|
|
"completions/mean_terminated_length": 778.703125,
|
|
"completions/min_length": 413.0,
|
|
"completions/min_terminated_length": 413.0,
|
|
"entropy": 0.8145867474377155,
|
|
"epoch": 0.0992063492063492,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.31640625,
|
|
"kl": 0.001623749420105014,
|
|
"learning_rate": 9.01360544217687e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 10536297.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.40999117493629456,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 175,
|
|
"step_time": 47.00463814288378
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1492.0,
|
|
"completions/max_terminated_length": 1492.0,
|
|
"completions/mean_length": 772.640625,
|
|
"completions/mean_terminated_length": 772.640625,
|
|
"completions/min_length": 322.0,
|
|
"completions/min_terminated_length": 322.0,
|
|
"entropy": 0.8795701172202826,
|
|
"epoch": 0.09977324263038549,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.0028222336040926166,
|
|
"learning_rate": 9.007936507936508e-07,
|
|
"loss": -0.003,
|
|
"num_tokens": 10595850.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5274578928947449,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 176,
|
|
"step_time": 58.35854543372989
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1640.0,
|
|
"completions/max_terminated_length": 1640.0,
|
|
"completions/mean_length": 749.1875,
|
|
"completions/mean_terminated_length": 749.1875,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.6966068465262651,
|
|
"epoch": 0.10034013605442177,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34375,
|
|
"kl": 0.0030260694547905587,
|
|
"learning_rate": 9.002267573696145e-07,
|
|
"loss": 0.0027,
|
|
"num_tokens": 10653542.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 177,
|
|
"step_time": 64.24640219751745
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1382.0,
|
|
"completions/max_terminated_length": 1382.0,
|
|
"completions/mean_length": 713.21875,
|
|
"completions/mean_terminated_length": 713.21875,
|
|
"completions/min_length": 243.0,
|
|
"completions/min_terminated_length": 243.0,
|
|
"entropy": 0.997425977140665,
|
|
"epoch": 0.10090702947845805,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.412109375,
|
|
"kl": 0.002245221330667846,
|
|
"learning_rate": 8.996598639455783e-07,
|
|
"loss": 0.0142,
|
|
"num_tokens": 10708476.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 178,
|
|
"step_time": 53.79487491585314
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1301.0,
|
|
"completions/max_terminated_length": 1301.0,
|
|
"completions/mean_length": 785.6875,
|
|
"completions/mean_terminated_length": 785.6875,
|
|
"completions/min_length": 360.0,
|
|
"completions/min_terminated_length": 360.0,
|
|
"entropy": 0.6753324847668409,
|
|
"epoch": 0.10147392290249434,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.326171875,
|
|
"kl": 0.0016486272761540022,
|
|
"learning_rate": 8.990929705215419e-07,
|
|
"loss": 0.0045,
|
|
"num_tokens": 10770408.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 179,
|
|
"step_time": 52.98196454253048
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1220.0,
|
|
"completions/max_terminated_length": 1220.0,
|
|
"completions/mean_length": 753.8125,
|
|
"completions/mean_terminated_length": 753.8125,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.7863912023603916,
|
|
"epoch": 0.10204081632653061,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.400390625,
|
|
"kl": 0.0020044324155605864,
|
|
"learning_rate": 8.985260770975056e-07,
|
|
"loss": 0.0256,
|
|
"num_tokens": 10831492.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 180,
|
|
"step_time": 49.65730780083686
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1241.0,
|
|
"completions/max_terminated_length": 1241.0,
|
|
"completions/mean_length": 763.765625,
|
|
"completions/mean_terminated_length": 763.765625,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.8663644455373287,
|
|
"epoch": 0.10260770975056689,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.0022015888171154074,
|
|
"learning_rate": 8.979591836734693e-07,
|
|
"loss": 0.0119,
|
|
"num_tokens": 10891677.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.53079754114151,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 181,
|
|
"step_time": 50.27756413444877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1282.0,
|
|
"completions/max_terminated_length": 1282.0,
|
|
"completions/mean_length": 672.859375,
|
|
"completions/mean_terminated_length": 672.859375,
|
|
"completions/min_length": 186.0,
|
|
"completions/min_terminated_length": 186.0,
|
|
"entropy": 0.8014075439423323,
|
|
"epoch": 0.10317460317460317,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.0026391869432700332,
|
|
"learning_rate": 8.973922902494331e-07,
|
|
"loss": -0.0063,
|
|
"num_tokens": 10944524.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6871842741966248,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 182,
|
|
"step_time": 49.701088985428214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1633.0,
|
|
"completions/max_terminated_length": 1633.0,
|
|
"completions/mean_length": 701.78125,
|
|
"completions/mean_terminated_length": 701.78125,
|
|
"completions/min_length": 140.0,
|
|
"completions/min_terminated_length": 140.0,
|
|
"entropy": 0.9052042663097382,
|
|
"epoch": 0.10374149659863946,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.58984375,
|
|
"kl": 0.0019432959816185758,
|
|
"learning_rate": 8.968253968253968e-07,
|
|
"loss": -0.0522,
|
|
"num_tokens": 10999318.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 183,
|
|
"step_time": 63.13254692032933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1192.0,
|
|
"completions/max_terminated_length": 1192.0,
|
|
"completions/mean_length": 774.4375,
|
|
"completions/mean_terminated_length": 774.4375,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.7909810151904821,
|
|
"epoch": 0.10430839002267574,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.61328125,
|
|
"kl": 0.0019899419603461865,
|
|
"learning_rate": 8.962585034013606e-07,
|
|
"loss": 0.0044,
|
|
"num_tokens": 11058786.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 184,
|
|
"step_time": 46.25606250949204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1666.0,
|
|
"completions/max_terminated_length": 1666.0,
|
|
"completions/mean_length": 810.3125,
|
|
"completions/mean_terminated_length": 810.3125,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.8799561206251383,
|
|
"epoch": 0.10487528344671201,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.002295402762683807,
|
|
"learning_rate": 8.956916099773242e-07,
|
|
"loss": 0.0031,
|
|
"num_tokens": 11120150.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 185,
|
|
"step_time": 65.20812787115574
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1911.0,
|
|
"completions/max_terminated_length": 1911.0,
|
|
"completions/mean_length": 819.484375,
|
|
"completions/mean_terminated_length": 819.484375,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.6690396033227444,
|
|
"epoch": 0.1054421768707483,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.0018117764229828026,
|
|
"learning_rate": 8.95124716553288e-07,
|
|
"loss": -0.0058,
|
|
"num_tokens": 11184109.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 186,
|
|
"step_time": 75.52050837129354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1695.0,
|
|
"completions/max_terminated_length": 1695.0,
|
|
"completions/mean_length": 776.59375,
|
|
"completions/mean_terminated_length": 776.59375,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.5842441543936729,
|
|
"epoch": 0.10600907029478458,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"kl": 0.0021664260639226995,
|
|
"learning_rate": 8.945578231292517e-07,
|
|
"loss": 0.014,
|
|
"num_tokens": 11243443.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 187,
|
|
"step_time": 65.45243280660361
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1191.0,
|
|
"completions/max_terminated_length": 1191.0,
|
|
"completions/mean_length": 705.875,
|
|
"completions/mean_terminated_length": 705.875,
|
|
"completions/min_length": 225.0,
|
|
"completions/min_terminated_length": 225.0,
|
|
"entropy": 0.6692053359001875,
|
|
"epoch": 0.10657596371882086,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.0031446133070858195,
|
|
"learning_rate": 8.939909297052153e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 11298731.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 188,
|
|
"step_time": 46.79448201600462
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1344.0,
|
|
"completions/max_terminated_length": 1344.0,
|
|
"completions/mean_length": 754.90625,
|
|
"completions/mean_terminated_length": 754.90625,
|
|
"completions/min_length": 286.0,
|
|
"completions/min_terminated_length": 286.0,
|
|
"entropy": 1.0256855934858322,
|
|
"epoch": 0.10714285714285714,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.0024256880205939524,
|
|
"learning_rate": 8.934240362811791e-07,
|
|
"loss": 0.0306,
|
|
"num_tokens": 11358741.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 189,
|
|
"step_time": 52.464261604472995
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1880.0,
|
|
"completions/max_terminated_length": 1880.0,
|
|
"completions/mean_length": 763.03125,
|
|
"completions/mean_terminated_length": 763.03125,
|
|
"completions/min_length": 268.0,
|
|
"completions/min_terminated_length": 268.0,
|
|
"entropy": 0.7950188089162111,
|
|
"epoch": 0.10770975056689343,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.0021833389000676107,
|
|
"learning_rate": 8.928571428571428e-07,
|
|
"loss": 0.0295,
|
|
"num_tokens": 11417503.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 190,
|
|
"step_time": 73.67372031044215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1137.0,
|
|
"completions/max_terminated_length": 1137.0,
|
|
"completions/mean_length": 695.25,
|
|
"completions/mean_terminated_length": 695.25,
|
|
"completions/min_length": 360.0,
|
|
"completions/min_terminated_length": 360.0,
|
|
"entropy": 0.7993376590311527,
|
|
"epoch": 0.1082766439909297,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58984375,
|
|
"kl": 0.002104887673340272,
|
|
"learning_rate": 8.922902494331065e-07,
|
|
"loss": 0.0093,
|
|
"num_tokens": 11474415.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 191,
|
|
"step_time": 44.51445526443422
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1473.0,
|
|
"completions/max_terminated_length": 1473.0,
|
|
"completions/mean_length": 791.46875,
|
|
"completions/mean_terminated_length": 791.46875,
|
|
"completions/min_length": 399.0,
|
|
"completions/min_terminated_length": 399.0,
|
|
"entropy": 0.7037545405328274,
|
|
"epoch": 0.10884353741496598,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2080078125,
|
|
"kl": 0.00409951618348714,
|
|
"learning_rate": 8.917233560090702e-07,
|
|
"loss": -0.0098,
|
|
"num_tokens": 11535149.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.33174845576286316,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 192,
|
|
"step_time": 56.72403950151056
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1284.0,
|
|
"completions/max_terminated_length": 1284.0,
|
|
"completions/mean_length": 755.765625,
|
|
"completions/mean_terminated_length": 755.765625,
|
|
"completions/min_length": 194.0,
|
|
"completions/min_terminated_length": 194.0,
|
|
"entropy": 0.7793229520320892,
|
|
"epoch": 0.10941043083900227,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.291015625,
|
|
"kl": 0.002601281084935181,
|
|
"learning_rate": 8.91156462585034e-07,
|
|
"loss": 0.0239,
|
|
"num_tokens": 11593254.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 193,
|
|
"step_time": 50.4504238832742
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1446.0,
|
|
"completions/max_terminated_length": 1446.0,
|
|
"completions/mean_length": 849.140625,
|
|
"completions/mean_terminated_length": 849.140625,
|
|
"completions/min_length": 495.0,
|
|
"completions/min_terminated_length": 495.0,
|
|
"entropy": 0.6648522336035967,
|
|
"epoch": 0.10997732426303855,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.002077874007227365,
|
|
"learning_rate": 8.905895691609977e-07,
|
|
"loss": 0.0106,
|
|
"num_tokens": 11656559.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 194,
|
|
"step_time": 56.28423589468002
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1553.0,
|
|
"completions/max_terminated_length": 1553.0,
|
|
"completions/mean_length": 688.515625,
|
|
"completions/mean_terminated_length": 688.515625,
|
|
"completions/min_length": 214.0,
|
|
"completions/min_terminated_length": 214.0,
|
|
"entropy": 0.8349446803331375,
|
|
"epoch": 0.11054421768707483,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.003143731963064056,
|
|
"learning_rate": 8.900226757369615e-07,
|
|
"loss": 0.0065,
|
|
"num_tokens": 11709888.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4714045226573944,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 195,
|
|
"step_time": 60.17381243593991
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1791.0,
|
|
"completions/max_terminated_length": 1791.0,
|
|
"completions/mean_length": 797.109375,
|
|
"completions/mean_terminated_length": 797.109375,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.9731405824422836,
|
|
"epoch": 0.1111111111111111,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6875,
|
|
"kl": 0.002387014923442621,
|
|
"learning_rate": 8.894557823129251e-07,
|
|
"loss": -0.0437,
|
|
"num_tokens": 11775207.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6412084698677063,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 196,
|
|
"step_time": 72.48106692917645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1246.0,
|
|
"completions/max_terminated_length": 1246.0,
|
|
"completions/mean_length": 653.96875,
|
|
"completions/mean_terminated_length": 653.96875,
|
|
"completions/min_length": 144.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"entropy": 0.8588339984416962,
|
|
"epoch": 0.1116780045351474,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5390625,
|
|
"kl": 0.0036632285846280865,
|
|
"learning_rate": 8.888888888888888e-07,
|
|
"loss": -0.0134,
|
|
"num_tokens": 11829389.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 197,
|
|
"step_time": 49.20649313181639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1651.0,
|
|
"completions/max_terminated_length": 1651.0,
|
|
"completions/mean_length": 700.578125,
|
|
"completions/mean_terminated_length": 700.578125,
|
|
"completions/min_length": 203.0,
|
|
"completions/min_terminated_length": 203.0,
|
|
"entropy": 0.8202382121235132,
|
|
"epoch": 0.11224489795918367,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.002230228237749543,
|
|
"learning_rate": 8.883219954648525e-07,
|
|
"loss": -0.0085,
|
|
"num_tokens": 11885482.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 198,
|
|
"step_time": 64.34397663455456
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1470.0,
|
|
"completions/mean_length": 768.25,
|
|
"completions/mean_terminated_length": 715.4285888671875,
|
|
"completions/min_length": 240.0,
|
|
"completions/min_terminated_length": 240.0,
|
|
"entropy": 0.7309875469654799,
|
|
"epoch": 0.11281179138321995,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.337890625,
|
|
"kl": 0.00237447653489653,
|
|
"learning_rate": 8.877551020408163e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 11943818.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.4038965702056885,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 199,
|
|
"step_time": 179.59818591643125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1305.0,
|
|
"completions/max_terminated_length": 1305.0,
|
|
"completions/mean_length": 728.984375,
|
|
"completions/mean_terminated_length": 728.984375,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.963052537292242,
|
|
"epoch": 0.11337868480725624,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"kl": 0.002730268613959197,
|
|
"learning_rate": 8.8718820861678e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 12000401.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 200,
|
|
"step_time": 51.364328576251864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1272.0,
|
|
"completions/max_terminated_length": 1272.0,
|
|
"completions/mean_length": 699.046875,
|
|
"completions/mean_terminated_length": 699.046875,
|
|
"completions/min_length": 400.0,
|
|
"completions/min_terminated_length": 400.0,
|
|
"entropy": 0.7668270245194435,
|
|
"epoch": 0.11394557823129252,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.0026685933116823435,
|
|
"learning_rate": 8.866213151927438e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 12055452.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 201,
|
|
"step_time": 50.06663680355996
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1189.0,
|
|
"completions/max_terminated_length": 1189.0,
|
|
"completions/mean_length": 669.21875,
|
|
"completions/mean_terminated_length": 669.21875,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.8293571621179581,
|
|
"epoch": 0.1145124716553288,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.002398704025836196,
|
|
"learning_rate": 8.860544217687075e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 12110410.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 202,
|
|
"step_time": 47.57376485038549
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1171.0,
|
|
"completions/max_terminated_length": 1171.0,
|
|
"completions/mean_length": 698.9375,
|
|
"completions/mean_terminated_length": 698.9375,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 1.000132780522108,
|
|
"epoch": 0.11507936507936507,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.314453125,
|
|
"kl": 0.002165637895814143,
|
|
"learning_rate": 8.854875283446712e-07,
|
|
"loss": 0.0056,
|
|
"num_tokens": 12164742.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 203,
|
|
"step_time": 46.32271307427436
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1144.0,
|
|
"completions/max_terminated_length": 1144.0,
|
|
"completions/mean_length": 746.25,
|
|
"completions/mean_terminated_length": 746.25,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.9319121092557907,
|
|
"epoch": 0.11564625850340136,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.365234375,
|
|
"kl": 0.002491012099198997,
|
|
"learning_rate": 8.849206349206348e-07,
|
|
"loss": -0.009,
|
|
"num_tokens": 12222470.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 204,
|
|
"step_time": 45.10238230600953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1662.0,
|
|
"completions/max_terminated_length": 1662.0,
|
|
"completions/mean_length": 806.640625,
|
|
"completions/mean_terminated_length": 806.640625,
|
|
"completions/min_length": 204.0,
|
|
"completions/min_terminated_length": 204.0,
|
|
"entropy": 0.6651069391518831,
|
|
"epoch": 0.11621315192743764,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.003674557614431251,
|
|
"learning_rate": 8.843537414965985e-07,
|
|
"loss": 0.0148,
|
|
"num_tokens": 12283039.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6423677802085876,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 205,
|
|
"step_time": 64.43609751947224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1808.0,
|
|
"completions/max_terminated_length": 1808.0,
|
|
"completions/mean_length": 818.53125,
|
|
"completions/mean_terminated_length": 818.53125,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.7762941531836987,
|
|
"epoch": 0.11678004535147392,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.003014502683072351,
|
|
"learning_rate": 8.837868480725623e-07,
|
|
"loss": -0.024,
|
|
"num_tokens": 12347177.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7339553236961365,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 206,
|
|
"step_time": 71.33777440246195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1298.0,
|
|
"completions/max_terminated_length": 1298.0,
|
|
"completions/mean_length": 698.671875,
|
|
"completions/mean_terminated_length": 698.671875,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.6892206575721502,
|
|
"epoch": 0.11734693877551021,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.004469722611247562,
|
|
"learning_rate": 8.83219954648526e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 12403292.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6416434645652771,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 207,
|
|
"step_time": 51.56914662569761
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1361.0,
|
|
"completions/max_terminated_length": 1361.0,
|
|
"completions/mean_length": 708.09375,
|
|
"completions/mean_terminated_length": 708.09375,
|
|
"completions/min_length": 370.0,
|
|
"completions/min_terminated_length": 370.0,
|
|
"entropy": 0.8025732468813658,
|
|
"epoch": 0.11791383219954649,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3984375,
|
|
"kl": 0.002947663400846068,
|
|
"learning_rate": 8.826530612244898e-07,
|
|
"loss": -0.0204,
|
|
"num_tokens": 12458490.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 208,
|
|
"step_time": 52.94201644603163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1789.0,
|
|
"completions/max_terminated_length": 1789.0,
|
|
"completions/mean_length": 809.0,
|
|
"completions/mean_terminated_length": 809.0,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.7231426276266575,
|
|
"epoch": 0.11848072562358276,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.0021686839136236813,
|
|
"learning_rate": 8.820861678004535e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 12519674.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 209,
|
|
"step_time": 68.73250697460026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1746.0,
|
|
"completions/max_terminated_length": 1746.0,
|
|
"completions/mean_length": 700.84375,
|
|
"completions/mean_terminated_length": 700.84375,
|
|
"completions/min_length": 292.0,
|
|
"completions/min_terminated_length": 292.0,
|
|
"entropy": 0.7687572091817856,
|
|
"epoch": 0.11904761904761904,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.003985284827649593,
|
|
"learning_rate": 8.815192743764172e-07,
|
|
"loss": 0.0077,
|
|
"num_tokens": 12574656.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 210,
|
|
"step_time": 67.19688565097749
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/max_terminated_length": 1376.0,
|
|
"completions/mean_length": 775.703125,
|
|
"completions/mean_terminated_length": 775.703125,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.7454958781599998,
|
|
"epoch": 0.11961451247165533,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.0025180759803333785,
|
|
"learning_rate": 8.809523809523809e-07,
|
|
"loss": 0.0144,
|
|
"num_tokens": 12634093.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 211,
|
|
"step_time": 54.07804360054433
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1331.0,
|
|
"completions/max_terminated_length": 1331.0,
|
|
"completions/mean_length": 844.921875,
|
|
"completions/mean_terminated_length": 844.921875,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.7186903860419989,
|
|
"epoch": 0.12018140589569161,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.0023014823818812147,
|
|
"learning_rate": 8.803854875283446e-07,
|
|
"loss": -0.0052,
|
|
"num_tokens": 12698072.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 212,
|
|
"step_time": 52.086543631739914
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1639.0,
|
|
"completions/max_terminated_length": 1639.0,
|
|
"completions/mean_length": 802.984375,
|
|
"completions/mean_terminated_length": 802.984375,
|
|
"completions/min_length": 191.0,
|
|
"completions/min_terminated_length": 191.0,
|
|
"entropy": 0.7671890594065189,
|
|
"epoch": 0.12074829931972789,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.004307562980102375,
|
|
"learning_rate": 8.798185941043083e-07,
|
|
"loss": 0.0089,
|
|
"num_tokens": 12761055.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5866720080375671,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 213,
|
|
"step_time": 64.068588770926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1307.0,
|
|
"completions/max_terminated_length": 1307.0,
|
|
"completions/mean_length": 792.921875,
|
|
"completions/mean_terminated_length": 792.921875,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.6857465747743845,
|
|
"epoch": 0.12131519274376418,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.0037605646284646355,
|
|
"learning_rate": 8.792517006802721e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 12822554.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 214,
|
|
"step_time": 53.09345087129623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1427.0,
|
|
"completions/max_terminated_length": 1427.0,
|
|
"completions/mean_length": 748.765625,
|
|
"completions/mean_terminated_length": 748.765625,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.7146262265741825,
|
|
"epoch": 0.12188208616780045,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.35546875,
|
|
"kl": 0.0030410429681069218,
|
|
"learning_rate": 8.786848072562358e-07,
|
|
"loss": -0.0116,
|
|
"num_tokens": 12880883.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4734388589859009,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 215,
|
|
"step_time": 58.4134808331728
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1176.0,
|
|
"completions/max_terminated_length": 1176.0,
|
|
"completions/mean_length": 782.625,
|
|
"completions/mean_terminated_length": 782.625,
|
|
"completions/min_length": 418.0,
|
|
"completions/min_terminated_length": 418.0,
|
|
"entropy": 1.0164642632007599,
|
|
"epoch": 0.12244897959183673,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.004625954588846071,
|
|
"learning_rate": 8.781179138321995e-07,
|
|
"loss": 0.0318,
|
|
"num_tokens": 12945563.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 216,
|
|
"step_time": 47.95185865368694
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1429.0,
|
|
"completions/max_terminated_length": 1429.0,
|
|
"completions/mean_length": 827.921875,
|
|
"completions/mean_terminated_length": 827.921875,
|
|
"completions/min_length": 429.0,
|
|
"completions/min_terminated_length": 429.0,
|
|
"entropy": 0.765678521245718,
|
|
"epoch": 0.12301587301587301,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.002401860627287533,
|
|
"learning_rate": 8.775510204081632e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 13008662.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 217,
|
|
"step_time": 56.30959102790803
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2061.0,
|
|
"completions/max_terminated_length": 2061.0,
|
|
"completions/mean_length": 891.515625,
|
|
"completions/mean_terminated_length": 891.515625,
|
|
"completions/min_length": 443.0,
|
|
"completions/min_terminated_length": 443.0,
|
|
"entropy": 0.7714218124747276,
|
|
"epoch": 0.1235827664399093,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.003679653353174217,
|
|
"learning_rate": 8.76984126984127e-07,
|
|
"loss": -0.021,
|
|
"num_tokens": 13076471.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 218,
|
|
"step_time": 80.66285381931812
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1234.0,
|
|
"completions/max_terminated_length": 1234.0,
|
|
"completions/mean_length": 693.625,
|
|
"completions/mean_terminated_length": 693.625,
|
|
"completions/min_length": 180.0,
|
|
"completions/min_terminated_length": 180.0,
|
|
"entropy": 0.789083743467927,
|
|
"epoch": 0.12414965986394558,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66796875,
|
|
"kl": 0.004488422848226037,
|
|
"learning_rate": 8.764172335600907e-07,
|
|
"loss": 0.0159,
|
|
"num_tokens": 13130831.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 219,
|
|
"step_time": 48.07829122338444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1337.0,
|
|
"completions/max_terminated_length": 1337.0,
|
|
"completions/mean_length": 709.78125,
|
|
"completions/mean_terminated_length": 709.78125,
|
|
"completions/min_length": 257.0,
|
|
"completions/min_terminated_length": 257.0,
|
|
"entropy": 1.0191886126995087,
|
|
"epoch": 0.12471655328798185,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.0039866922234068625,
|
|
"learning_rate": 8.758503401360544e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 13186977.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 220,
|
|
"step_time": 51.81976623181254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1464.0,
|
|
"completions/max_terminated_length": 1464.0,
|
|
"completions/mean_length": 823.171875,
|
|
"completions/mean_terminated_length": 823.171875,
|
|
"completions/min_length": 475.0,
|
|
"completions/min_terminated_length": 475.0,
|
|
"entropy": 0.7741623353213072,
|
|
"epoch": 0.12528344671201813,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"kl": 0.004347501919255592,
|
|
"learning_rate": 8.752834467120181e-07,
|
|
"loss": -0.0022,
|
|
"num_tokens": 13250300.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 221,
|
|
"step_time": 56.70674694888294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1369.0,
|
|
"completions/max_terminated_length": 1369.0,
|
|
"completions/mean_length": 766.46875,
|
|
"completions/mean_terminated_length": 766.46875,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 0.862474599853158,
|
|
"epoch": 0.12585034013605442,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.0032451720435346942,
|
|
"learning_rate": 8.747165532879818e-07,
|
|
"loss": -0.0089,
|
|
"num_tokens": 13311818.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 222,
|
|
"step_time": 54.62590129673481
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1355.0,
|
|
"completions/max_terminated_length": 1355.0,
|
|
"completions/mean_length": 647.515625,
|
|
"completions/mean_terminated_length": 647.515625,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.9779625628143549,
|
|
"epoch": 0.1264172335600907,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5390625,
|
|
"kl": 0.004331238793383818,
|
|
"learning_rate": 8.741496598639455e-07,
|
|
"loss": 0.0183,
|
|
"num_tokens": 13362787.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 223,
|
|
"step_time": 52.470416551455855
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1475.0,
|
|
"completions/max_terminated_length": 1475.0,
|
|
"completions/mean_length": 722.671875,
|
|
"completions/mean_terminated_length": 722.671875,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.7243257965892553,
|
|
"epoch": 0.12698412698412698,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.004345040906628128,
|
|
"learning_rate": 8.735827664399093e-07,
|
|
"loss": 0.0078,
|
|
"num_tokens": 13418782.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7622999548912048,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 224,
|
|
"step_time": 57.42830611113459
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1300.0,
|
|
"completions/max_terminated_length": 1300.0,
|
|
"completions/mean_length": 712.6875,
|
|
"completions/mean_terminated_length": 712.6875,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.9633335061371326,
|
|
"epoch": 0.12755102040816327,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.003034610257600434,
|
|
"learning_rate": 8.73015873015873e-07,
|
|
"loss": -0.007,
|
|
"num_tokens": 13480690.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.653468668460846,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 225,
|
|
"step_time": 54.13289635069668
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2195.0,
|
|
"completions/max_terminated_length": 2195.0,
|
|
"completions/mean_length": 703.703125,
|
|
"completions/mean_terminated_length": 703.703125,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.6303536836057901,
|
|
"epoch": 0.12811791383219956,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.004575064493110403,
|
|
"learning_rate": 8.724489795918368e-07,
|
|
"loss": 0.016,
|
|
"num_tokens": 13535311.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 226,
|
|
"step_time": 85.67797674704343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1494.0,
|
|
"completions/max_terminated_length": 1494.0,
|
|
"completions/mean_length": 750.703125,
|
|
"completions/mean_terminated_length": 750.703125,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.8935067839920521,
|
|
"epoch": 0.12868480725623582,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.005571341884206049,
|
|
"learning_rate": 8.718820861678005e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 13593484.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 227,
|
|
"step_time": 58.77007927093655
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 708.484375,
|
|
"completions/mean_terminated_length": 708.484375,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.8600424602627754,
|
|
"epoch": 0.1292517006802721,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.004000057917437516,
|
|
"learning_rate": 8.71315192743764e-07,
|
|
"loss": -0.0138,
|
|
"num_tokens": 13648171.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 228,
|
|
"step_time": 48.869283626787364
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1423.0,
|
|
"completions/max_terminated_length": 1423.0,
|
|
"completions/mean_length": 818.390625,
|
|
"completions/mean_terminated_length": 818.390625,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.6593409366905689,
|
|
"epoch": 0.12981859410430838,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.0035780292018898763,
|
|
"learning_rate": 8.707482993197278e-07,
|
|
"loss": 0.0011,
|
|
"num_tokens": 13710476.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6782841682434082,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 229,
|
|
"step_time": 55.18314048089087
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1530.0,
|
|
"completions/max_terminated_length": 1530.0,
|
|
"completions/mean_length": 649.5625,
|
|
"completions/mean_terminated_length": 649.5625,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 1.0032381936907768,
|
|
"epoch": 0.13038548752834467,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.004240339570969809,
|
|
"learning_rate": 8.701814058956915e-07,
|
|
"loss": 0.0016,
|
|
"num_tokens": 13762320.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 230,
|
|
"step_time": 60.95228326693177
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1380.0,
|
|
"completions/max_terminated_length": 1380.0,
|
|
"completions/mean_length": 783.96875,
|
|
"completions/mean_terminated_length": 783.96875,
|
|
"completions/min_length": 242.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"entropy": 0.7472527362406254,
|
|
"epoch": 0.13095238095238096,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.005484887624334078,
|
|
"learning_rate": 8.696145124716553e-07,
|
|
"loss": 0.0353,
|
|
"num_tokens": 13821494.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 231,
|
|
"step_time": 54.40403426066041
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1246.0,
|
|
"completions/max_terminated_length": 1246.0,
|
|
"completions/mean_length": 776.765625,
|
|
"completions/mean_terminated_length": 776.765625,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 0.7287231758236885,
|
|
"epoch": 0.13151927437641722,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"kl": 0.003164727291732561,
|
|
"learning_rate": 8.69047619047619e-07,
|
|
"loss": 0.0172,
|
|
"num_tokens": 13882103.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 232,
|
|
"step_time": 49.35764399077743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1446.0,
|
|
"completions/max_terminated_length": 1446.0,
|
|
"completions/mean_length": 740.796875,
|
|
"completions/mean_terminated_length": 740.796875,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.8617698810994625,
|
|
"epoch": 0.1320861678004535,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.0035138000966981053,
|
|
"learning_rate": 8.684807256235828e-07,
|
|
"loss": 0.0197,
|
|
"num_tokens": 13939714.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 233,
|
|
"step_time": 55.777379545383155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1309.0,
|
|
"completions/max_terminated_length": 1309.0,
|
|
"completions/mean_length": 686.359375,
|
|
"completions/mean_terminated_length": 686.359375,
|
|
"completions/min_length": 204.0,
|
|
"completions/min_terminated_length": 204.0,
|
|
"entropy": 1.0610783658921719,
|
|
"epoch": 0.1326530612244898,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.3515625,
|
|
"kl": 0.006404620158718899,
|
|
"learning_rate": 8.679138321995464e-07,
|
|
"loss": 0.002,
|
|
"num_tokens": 13994833.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 234,
|
|
"step_time": 51.19552284292877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1355.0,
|
|
"completions/mean_length": 816.0,
|
|
"completions/mean_terminated_length": 763.9365234375,
|
|
"completions/min_length": 382.0,
|
|
"completions/min_terminated_length": 382.0,
|
|
"entropy": 0.9454163573682308,
|
|
"epoch": 0.13321995464852607,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.004431686924363021,
|
|
"learning_rate": 8.673469387755102e-07,
|
|
"loss": -0.0152,
|
|
"num_tokens": 14061857.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5729843378067017,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 235,
|
|
"step_time": 184.50176417734474
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1570.0,
|
|
"completions/max_terminated_length": 1570.0,
|
|
"completions/mean_length": 843.78125,
|
|
"completions/mean_terminated_length": 843.78125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.7127547487616539,
|
|
"epoch": 0.13378684807256236,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.337890625,
|
|
"kl": 0.007691268219787162,
|
|
"learning_rate": 8.667800453514738e-07,
|
|
"loss": -0.0137,
|
|
"num_tokens": 14129947.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 236,
|
|
"step_time": 61.732176676392555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1457.0,
|
|
"completions/max_terminated_length": 1457.0,
|
|
"completions/mean_length": 693.15625,
|
|
"completions/mean_terminated_length": 693.15625,
|
|
"completions/min_length": 305.0,
|
|
"completions/min_terminated_length": 305.0,
|
|
"entropy": 0.8555677160620689,
|
|
"epoch": 0.13435374149659865,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.010487258521607146,
|
|
"learning_rate": 8.662131519274376e-07,
|
|
"loss": 0.0315,
|
|
"num_tokens": 14184837.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 237,
|
|
"step_time": 56.15231809951365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 4086.0,
|
|
"completions/max_terminated_length": 4086.0,
|
|
"completions/mean_length": 884.59375,
|
|
"completions/mean_terminated_length": 884.59375,
|
|
"completions/min_length": 380.0,
|
|
"completions/min_terminated_length": 380.0,
|
|
"entropy": 0.6391396187245846,
|
|
"epoch": 0.1349206349206349,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.33984375,
|
|
"kl": 0.003827203137916513,
|
|
"learning_rate": 8.656462585034013e-07,
|
|
"loss": 0.0065,
|
|
"num_tokens": 14251563.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4753340482711792,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 238,
|
|
"step_time": 179.5403899597004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2424.0,
|
|
"completions/max_terminated_length": 2424.0,
|
|
"completions/mean_length": 756.859375,
|
|
"completions/mean_terminated_length": 756.859375,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.9389139264822006,
|
|
"epoch": 0.1354875283446712,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"kl": 0.0035379417677177116,
|
|
"learning_rate": 8.650793650793651e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 14309914.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 239,
|
|
"step_time": 94.99134722538292
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1493.0,
|
|
"completions/max_terminated_length": 1493.0,
|
|
"completions/mean_length": 785.65625,
|
|
"completions/mean_terminated_length": 785.65625,
|
|
"completions/min_length": 292.0,
|
|
"completions/min_terminated_length": 292.0,
|
|
"entropy": 0.8213918693363667,
|
|
"epoch": 0.1360544217687075,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.396484375,
|
|
"kl": 0.004000949513283558,
|
|
"learning_rate": 8.645124716553288e-07,
|
|
"loss": 0.0167,
|
|
"num_tokens": 14369980.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 240,
|
|
"step_time": 58.402712346054614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2243.0,
|
|
"completions/max_terminated_length": 2243.0,
|
|
"completions/mean_length": 775.125,
|
|
"completions/mean_terminated_length": 775.125,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.6468172837048769,
|
|
"epoch": 0.13662131519274376,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.337890625,
|
|
"kl": 0.009005385334603488,
|
|
"learning_rate": 8.639455782312925e-07,
|
|
"loss": 0.0185,
|
|
"num_tokens": 14429644.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 241,
|
|
"step_time": 89.81586427427828
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1471.0,
|
|
"completions/max_terminated_length": 1471.0,
|
|
"completions/mean_length": 817.703125,
|
|
"completions/mean_terminated_length": 817.703125,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.7675363086163998,
|
|
"epoch": 0.13718820861678005,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.003092183542321436,
|
|
"learning_rate": 8.633786848072562e-07,
|
|
"loss": -0.0005,
|
|
"num_tokens": 14491881.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 242,
|
|
"step_time": 57.815207845531404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/max_terminated_length": 1283.0,
|
|
"completions/mean_length": 643.5625,
|
|
"completions/mean_terminated_length": 643.5625,
|
|
"completions/min_length": 206.0,
|
|
"completions/min_terminated_length": 206.0,
|
|
"entropy": 0.854773510247469,
|
|
"epoch": 0.1377551020408163,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66015625,
|
|
"kl": 0.004636547317204531,
|
|
"learning_rate": 8.6281179138322e-07,
|
|
"loss": -0.0201,
|
|
"num_tokens": 14544165.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 243,
|
|
"step_time": 51.11892264150083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1606.0,
|
|
"completions/max_terminated_length": 1606.0,
|
|
"completions/mean_length": 752.578125,
|
|
"completions/mean_terminated_length": 752.578125,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.7487936839461327,
|
|
"epoch": 0.1383219954648526,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.0038994086789898574,
|
|
"learning_rate": 8.622448979591836e-07,
|
|
"loss": -0.0097,
|
|
"num_tokens": 14602258.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6782841682434082,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 244,
|
|
"step_time": 62.81886448804289
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1134.0,
|
|
"completions/max_terminated_length": 1134.0,
|
|
"completions/mean_length": 730.46875,
|
|
"completions/mean_terminated_length": 730.46875,
|
|
"completions/min_length": 387.0,
|
|
"completions/min_terminated_length": 387.0,
|
|
"entropy": 0.8416277430951595,
|
|
"epoch": 0.1388888888888889,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40625,
|
|
"kl": 0.004320491309044883,
|
|
"learning_rate": 8.616780045351474e-07,
|
|
"loss": -0.0115,
|
|
"num_tokens": 14662184.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 245,
|
|
"step_time": 46.775943524204195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1520.0,
|
|
"completions/max_terminated_length": 1520.0,
|
|
"completions/mean_length": 813.6875,
|
|
"completions/mean_terminated_length": 813.6875,
|
|
"completions/min_length": 332.0,
|
|
"completions/min_terminated_length": 332.0,
|
|
"entropy": 0.8341892696917057,
|
|
"epoch": 0.13945578231292516,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.330078125,
|
|
"kl": 0.005075214154203422,
|
|
"learning_rate": 8.611111111111111e-07,
|
|
"loss": 0.0178,
|
|
"num_tokens": 14726332.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 246,
|
|
"step_time": 59.55118718650192
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1229.0,
|
|
"completions/max_terminated_length": 1229.0,
|
|
"completions/mean_length": 768.390625,
|
|
"completions/mean_terminated_length": 768.390625,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.8386390097439289,
|
|
"epoch": 0.14002267573696145,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25390625,
|
|
"kl": 0.004386021581012756,
|
|
"learning_rate": 8.605442176870747e-07,
|
|
"loss": -0.002,
|
|
"num_tokens": 14790253.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 247,
|
|
"step_time": 50.37566922232509
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1222.0,
|
|
"completions/max_terminated_length": 1222.0,
|
|
"completions/mean_length": 671.953125,
|
|
"completions/mean_terminated_length": 671.953125,
|
|
"completions/min_length": 220.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"entropy": 0.7530262172222137,
|
|
"epoch": 0.14058956916099774,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.63671875,
|
|
"kl": 0.003375259351741988,
|
|
"learning_rate": 8.599773242630385e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 14842218.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 248,
|
|
"step_time": 47.82787063345313
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1987.0,
|
|
"completions/max_terminated_length": 1987.0,
|
|
"completions/mean_length": 831.53125,
|
|
"completions/mean_terminated_length": 831.53125,
|
|
"completions/min_length": 401.0,
|
|
"completions/min_terminated_length": 401.0,
|
|
"entropy": 0.7898271754384041,
|
|
"epoch": 0.141156462585034,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.003935490451112855,
|
|
"learning_rate": 8.594104308390022e-07,
|
|
"loss": -0.0057,
|
|
"num_tokens": 14907324.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 249,
|
|
"step_time": 78.51457949448377
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1259.0,
|
|
"completions/max_terminated_length": 1259.0,
|
|
"completions/mean_length": 710.53125,
|
|
"completions/mean_terminated_length": 710.53125,
|
|
"completions/min_length": 383.0,
|
|
"completions/min_terminated_length": 383.0,
|
|
"entropy": 0.8600624240934849,
|
|
"epoch": 0.1417233560090703,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.0037620200600940734,
|
|
"learning_rate": 8.58843537414966e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 14962966.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 250,
|
|
"step_time": 49.08725009299815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1145.0,
|
|
"completions/mean_length": 699.953125,
|
|
"completions/mean_terminated_length": 646.0476684570312,
|
|
"completions/min_length": 324.0,
|
|
"completions/min_terminated_length": 324.0,
|
|
"entropy": 1.1695469804108143,
|
|
"epoch": 0.14229024943310659,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.007454679012880661,
|
|
"learning_rate": 8.582766439909297e-07,
|
|
"loss": 0.0073,
|
|
"num_tokens": 15018755.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 251,
|
|
"step_time": 180.536724999547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1471.0,
|
|
"completions/max_terminated_length": 1471.0,
|
|
"completions/mean_length": 689.578125,
|
|
"completions/mean_terminated_length": 689.578125,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.8454550709575415,
|
|
"epoch": 0.14285714285714285,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.015838683204492554,
|
|
"learning_rate": 8.577097505668934e-07,
|
|
"loss": -0.0188,
|
|
"num_tokens": 15073768.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 252,
|
|
"step_time": 56.604044997133315
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1138.0,
|
|
"completions/max_terminated_length": 1138.0,
|
|
"completions/mean_length": 725.8125,
|
|
"completions/mean_terminated_length": 725.8125,
|
|
"completions/min_length": 428.0,
|
|
"completions/min_terminated_length": 428.0,
|
|
"entropy": 0.9270643517374992,
|
|
"epoch": 0.14342403628117914,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.64453125,
|
|
"kl": 0.006181689605000429,
|
|
"learning_rate": 8.57142857142857e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 15129764.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 253,
|
|
"step_time": 43.90463090222329
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1675.0,
|
|
"completions/max_terminated_length": 1675.0,
|
|
"completions/mean_length": 745.328125,
|
|
"completions/mean_terminated_length": 745.328125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.749716816470027,
|
|
"epoch": 0.14399092970521543,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.007256982797116507,
|
|
"learning_rate": 8.565759637188208e-07,
|
|
"loss": -0.015,
|
|
"num_tokens": 15188457.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 254,
|
|
"step_time": 66.19093087594956
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1119.0,
|
|
"completions/max_terminated_length": 1119.0,
|
|
"completions/mean_length": 724.421875,
|
|
"completions/mean_terminated_length": 724.421875,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.8893743939697742,
|
|
"epoch": 0.1445578231292517,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.007954366330523044,
|
|
"learning_rate": 8.560090702947845e-07,
|
|
"loss": -0.0089,
|
|
"num_tokens": 15244892.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 255,
|
|
"step_time": 44.45121990144253
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1564.0,
|
|
"completions/max_terminated_length": 1564.0,
|
|
"completions/mean_length": 738.25,
|
|
"completions/mean_terminated_length": 738.25,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 1.016543436795473,
|
|
"epoch": 0.14512471655328799,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"kl": 0.0035384253205847926,
|
|
"learning_rate": 8.554421768707483e-07,
|
|
"loss": -0.0028,
|
|
"num_tokens": 15304892.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 256,
|
|
"step_time": 63.003812201321125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1463.0,
|
|
"completions/max_terminated_length": 1463.0,
|
|
"completions/mean_length": 674.3125,
|
|
"completions/mean_terminated_length": 674.3125,
|
|
"completions/min_length": 226.0,
|
|
"completions/min_terminated_length": 226.0,
|
|
"entropy": 0.8456391058862209,
|
|
"epoch": 0.14569160997732428,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.007904115656856447,
|
|
"learning_rate": 8.54875283446712e-07,
|
|
"loss": 0.0318,
|
|
"num_tokens": 15360568.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 257,
|
|
"step_time": 57.653264158405364
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1193.0,
|
|
"completions/max_terminated_length": 1193.0,
|
|
"completions/mean_length": 683.625,
|
|
"completions/mean_terminated_length": 683.625,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.7597847003489733,
|
|
"epoch": 0.14625850340136054,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.006292103018495254,
|
|
"learning_rate": 8.543083900226758e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 15413344.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6422230005264282,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 258,
|
|
"step_time": 46.00685829296708
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1294.0,
|
|
"completions/max_terminated_length": 1294.0,
|
|
"completions/mean_length": 794.0625,
|
|
"completions/mean_terminated_length": 794.0625,
|
|
"completions/min_length": 500.0,
|
|
"completions/min_terminated_length": 500.0,
|
|
"entropy": 0.7875265162438154,
|
|
"epoch": 0.14682539682539683,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.0028908635285915807,
|
|
"learning_rate": 8.537414965986394e-07,
|
|
"loss": -0.0122,
|
|
"num_tokens": 15474628.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 259,
|
|
"step_time": 51.2000568471849
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1213.0,
|
|
"completions/max_terminated_length": 1213.0,
|
|
"completions/mean_length": 707.296875,
|
|
"completions/mean_terminated_length": 707.296875,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.8221424091607332,
|
|
"epoch": 0.1473922902494331,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.00474485175072914,
|
|
"learning_rate": 8.531746031746031e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 15529135.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6993034482002258,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 260,
|
|
"step_time": 47.618589179590344
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1819.0,
|
|
"completions/max_terminated_length": 1819.0,
|
|
"completions/mean_length": 734.65625,
|
|
"completions/mean_terminated_length": 734.65625,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.7079410217702389,
|
|
"epoch": 0.14795918367346939,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.65625,
|
|
"kl": 0.0045277264434844255,
|
|
"learning_rate": 8.526077097505668e-07,
|
|
"loss": -0.0294,
|
|
"num_tokens": 15587033.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.7676090598106384,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 261,
|
|
"step_time": 71.04479949083179
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1280.0,
|
|
"completions/max_terminated_length": 1280.0,
|
|
"completions/mean_length": 783.171875,
|
|
"completions/mean_terminated_length": 783.171875,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.9606173038482666,
|
|
"epoch": 0.14852607709750568,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.244140625,
|
|
"kl": 0.009063210411113687,
|
|
"learning_rate": 8.520408163265306e-07,
|
|
"loss": 0.0147,
|
|
"num_tokens": 15648540.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 262,
|
|
"step_time": 50.15093679446727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1643.0,
|
|
"completions/max_terminated_length": 1643.0,
|
|
"completions/mean_length": 833.578125,
|
|
"completions/mean_terminated_length": 833.578125,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.7215248327702284,
|
|
"epoch": 0.14909297052154194,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.006525891963974573,
|
|
"learning_rate": 8.514739229024943e-07,
|
|
"loss": -0.0078,
|
|
"num_tokens": 15712585.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 263,
|
|
"step_time": 63.85956168174744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1282.0,
|
|
"completions/max_terminated_length": 1282.0,
|
|
"completions/mean_length": 717.484375,
|
|
"completions/mean_terminated_length": 717.484375,
|
|
"completions/min_length": 466.0,
|
|
"completions/min_terminated_length": 466.0,
|
|
"entropy": 1.0237734578549862,
|
|
"epoch": 0.14965986394557823,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.357421875,
|
|
"kl": 0.007683624091441743,
|
|
"learning_rate": 8.509070294784581e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 15768728.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 264,
|
|
"step_time": 49.85610455926508
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1148.0,
|
|
"completions/max_terminated_length": 1148.0,
|
|
"completions/mean_length": 654.046875,
|
|
"completions/mean_terminated_length": 654.046875,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 1.0060841478407383,
|
|
"epoch": 0.15022675736961452,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55078125,
|
|
"kl": 0.011602386686718091,
|
|
"learning_rate": 8.503401360544217e-07,
|
|
"loss": -0.0132,
|
|
"num_tokens": 15821251.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.8273220658302307,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 265,
|
|
"step_time": 45.338741284795105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 973.0,
|
|
"completions/max_terminated_length": 973.0,
|
|
"completions/mean_length": 668.828125,
|
|
"completions/mean_terminated_length": 668.828125,
|
|
"completions/min_length": 294.0,
|
|
"completions/min_terminated_length": 294.0,
|
|
"entropy": 1.005034700036049,
|
|
"epoch": 0.15079365079365079,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.008998857665574178,
|
|
"learning_rate": 8.497732426303855e-07,
|
|
"loss": -0.0007,
|
|
"num_tokens": 15873336.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 266,
|
|
"step_time": 38.137159804813564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 717.421875,
|
|
"completions/mean_terminated_length": 717.421875,
|
|
"completions/min_length": 253.0,
|
|
"completions/min_terminated_length": 253.0,
|
|
"entropy": 0.7188418917357922,
|
|
"epoch": 0.15136054421768708,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.007004673054325394,
|
|
"learning_rate": 8.492063492063492e-07,
|
|
"loss": -0.0062,
|
|
"num_tokens": 15930243.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 267,
|
|
"step_time": 53.01031909137964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1437.0,
|
|
"completions/max_terminated_length": 1437.0,
|
|
"completions/mean_length": 695.40625,
|
|
"completions/mean_terminated_length": 695.40625,
|
|
"completions/min_length": 236.0,
|
|
"completions/min_terminated_length": 236.0,
|
|
"entropy": 0.7823151145130396,
|
|
"epoch": 0.15192743764172337,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.003142316978482995,
|
|
"learning_rate": 8.486394557823128e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 15988117.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 268,
|
|
"step_time": 56.694628211669624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1636.0,
|
|
"completions/max_terminated_length": 1636.0,
|
|
"completions/mean_length": 605.09375,
|
|
"completions/mean_terminated_length": 605.09375,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 0.8527481835335493,
|
|
"epoch": 0.15249433106575963,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.72265625,
|
|
"kl": 0.004755638008646201,
|
|
"learning_rate": 8.480725623582766e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 16038539.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 269,
|
|
"step_time": 63.849821055307984
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1465.0,
|
|
"completions/max_terminated_length": 1465.0,
|
|
"completions/mean_length": 727.6875,
|
|
"completions/mean_terminated_length": 727.6875,
|
|
"completions/min_length": 253.0,
|
|
"completions/min_terminated_length": 253.0,
|
|
"entropy": 0.8521798010915518,
|
|
"epoch": 0.15306122448979592,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.010075078469526488,
|
|
"learning_rate": 8.475056689342403e-07,
|
|
"loss": -0.0073,
|
|
"num_tokens": 16096135.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 270,
|
|
"step_time": 57.340402787551284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1408.0,
|
|
"completions/max_terminated_length": 1408.0,
|
|
"completions/mean_length": 809.09375,
|
|
"completions/mean_terminated_length": 809.09375,
|
|
"completions/min_length": 264.0,
|
|
"completions/min_terminated_length": 264.0,
|
|
"entropy": 0.9269996285438538,
|
|
"epoch": 0.1536281179138322,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.010795337875606492,
|
|
"learning_rate": 8.469387755102041e-07,
|
|
"loss": 0.0172,
|
|
"num_tokens": 16158221.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6662480235099792,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 271,
|
|
"step_time": 55.47354524489492
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1257.0,
|
|
"completions/max_terminated_length": 1257.0,
|
|
"completions/mean_length": 714.09375,
|
|
"completions/mean_terminated_length": 714.09375,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 0.9163622744381428,
|
|
"epoch": 0.15419501133786848,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.004687610104156192,
|
|
"learning_rate": 8.463718820861677e-07,
|
|
"loss": 0.0279,
|
|
"num_tokens": 16214643.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.7123050093650818,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 272,
|
|
"step_time": 49.16124189272523
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1902.0,
|
|
"completions/max_terminated_length": 1902.0,
|
|
"completions/mean_length": 791.703125,
|
|
"completions/mean_terminated_length": 791.703125,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 0.7695765160024166,
|
|
"epoch": 0.15476190476190477,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.004302407833165489,
|
|
"learning_rate": 8.458049886621315e-07,
|
|
"loss": -0.0395,
|
|
"num_tokens": 16275944.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 273,
|
|
"step_time": 74.4036480858922
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/max_terminated_length": 1376.0,
|
|
"completions/mean_length": 726.25,
|
|
"completions/mean_terminated_length": 726.25,
|
|
"completions/min_length": 246.0,
|
|
"completions/min_terminated_length": 246.0,
|
|
"entropy": 0.7283195722848177,
|
|
"epoch": 0.15532879818594103,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.65234375,
|
|
"kl": 0.008357068902114406,
|
|
"learning_rate": 8.452380952380952e-07,
|
|
"loss": 0.0142,
|
|
"num_tokens": 16332280.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 274,
|
|
"step_time": 53.547890301793814
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1329.0,
|
|
"completions/max_terminated_length": 1329.0,
|
|
"completions/mean_length": 689.5625,
|
|
"completions/mean_terminated_length": 689.5625,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.7559501770883799,
|
|
"epoch": 0.15589569160997732,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.017642906168475747,
|
|
"learning_rate": 8.44671201814059e-07,
|
|
"loss": 0.0387,
|
|
"num_tokens": 16385708.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7194505929946899,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 275,
|
|
"step_time": 51.140076864510775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1377.0,
|
|
"completions/max_terminated_length": 1377.0,
|
|
"completions/mean_length": 801.453125,
|
|
"completions/mean_terminated_length": 801.453125,
|
|
"completions/min_length": 286.0,
|
|
"completions/min_terminated_length": 286.0,
|
|
"entropy": 0.76519157551229,
|
|
"epoch": 0.1564625850340136,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.005709742763428949,
|
|
"learning_rate": 8.441043083900226e-07,
|
|
"loss": -0.0031,
|
|
"num_tokens": 16448001.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.40999117493629456,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 276,
|
|
"step_time": 53.25250900071114
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1350.0,
|
|
"completions/max_terminated_length": 1350.0,
|
|
"completions/mean_length": 624.234375,
|
|
"completions/mean_terminated_length": 624.234375,
|
|
"completions/min_length": 194.0,
|
|
"completions/min_terminated_length": 194.0,
|
|
"entropy": 0.7580607645213604,
|
|
"epoch": 0.15702947845804988,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.73828125,
|
|
"kl": 0.01761455085943453,
|
|
"learning_rate": 8.435374149659864e-07,
|
|
"loss": -0.0033,
|
|
"num_tokens": 16499128.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7295067310333252,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 277,
|
|
"step_time": 52.124403435736895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1474.0,
|
|
"completions/max_terminated_length": 1474.0,
|
|
"completions/mean_length": 888.296875,
|
|
"completions/mean_terminated_length": 888.296875,
|
|
"completions/min_length": 364.0,
|
|
"completions/min_terminated_length": 364.0,
|
|
"entropy": 0.7863531224429607,
|
|
"epoch": 0.15759637188208617,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.006082025865907781,
|
|
"learning_rate": 8.4297052154195e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 16567819.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 278,
|
|
"step_time": 58.55780995916575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1131.0,
|
|
"completions/max_terminated_length": 1131.0,
|
|
"completions/mean_length": 710.3125,
|
|
"completions/mean_terminated_length": 710.3125,
|
|
"completions/min_length": 414.0,
|
|
"completions/min_terminated_length": 414.0,
|
|
"entropy": 0.8654900714755058,
|
|
"epoch": 0.15816326530612246,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.004806106633623131,
|
|
"learning_rate": 8.424036281179138e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 16622871.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 279,
|
|
"step_time": 44.18114794231951
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1269.0,
|
|
"completions/max_terminated_length": 1269.0,
|
|
"completions/mean_length": 694.734375,
|
|
"completions/mean_terminated_length": 694.734375,
|
|
"completions/min_length": 280.0,
|
|
"completions/min_terminated_length": 280.0,
|
|
"entropy": 1.0109095703810453,
|
|
"epoch": 0.15873015873015872,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.62890625,
|
|
"kl": 0.007879828044679016,
|
|
"learning_rate": 8.418367346938775e-07,
|
|
"loss": 0.0182,
|
|
"num_tokens": 16676726.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 280,
|
|
"step_time": 49.75455924682319
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1565.0,
|
|
"completions/max_terminated_length": 1565.0,
|
|
"completions/mean_length": 772.671875,
|
|
"completions/mean_terminated_length": 772.671875,
|
|
"completions/min_length": 238.0,
|
|
"completions/min_terminated_length": 238.0,
|
|
"entropy": 0.7377403303980827,
|
|
"epoch": 0.159297052154195,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.006127516506239772,
|
|
"learning_rate": 8.412698412698413e-07,
|
|
"loss": -0.008,
|
|
"num_tokens": 16736313.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 281,
|
|
"step_time": 61.17428585886955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1397.0,
|
|
"completions/max_terminated_length": 1397.0,
|
|
"completions/mean_length": 787.421875,
|
|
"completions/mean_terminated_length": 787.421875,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.6864061839878559,
|
|
"epoch": 0.1598639455782313,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5390625,
|
|
"kl": 0.0049530532414792106,
|
|
"learning_rate": 8.40702947845805e-07,
|
|
"loss": 0.0018,
|
|
"num_tokens": 16796828.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 282,
|
|
"step_time": 54.73281847964972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1169.0,
|
|
"completions/max_terminated_length": 1169.0,
|
|
"completions/mean_length": 652.75,
|
|
"completions/mean_terminated_length": 652.75,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.7306120228022337,
|
|
"epoch": 0.16043083900226757,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.013816178601700813,
|
|
"learning_rate": 8.401360544217688e-07,
|
|
"loss": -0.0133,
|
|
"num_tokens": 16850964.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6542273163795471,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 283,
|
|
"step_time": 45.850550681352615
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1697.0,
|
|
"completions/max_terminated_length": 1697.0,
|
|
"completions/mean_length": 670.9375,
|
|
"completions/mean_terminated_length": 670.9375,
|
|
"completions/min_length": 229.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"entropy": 0.9432363584637642,
|
|
"epoch": 0.16099773242630386,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.0031036645887070335,
|
|
"learning_rate": 8.395691609977323e-07,
|
|
"loss": -0.0136,
|
|
"num_tokens": 16903624.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7099072337150574,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 284,
|
|
"step_time": 65.68877389188856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1536.0,
|
|
"completions/mean_length": 724.8125,
|
|
"completions/mean_terminated_length": 724.8125,
|
|
"completions/min_length": 192.0,
|
|
"completions/min_terminated_length": 192.0,
|
|
"entropy": 0.7398256752640009,
|
|
"epoch": 0.16156462585034015,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.0050950217846548185,
|
|
"learning_rate": 8.39002267573696e-07,
|
|
"loss": -0.01,
|
|
"num_tokens": 16960140.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 285,
|
|
"step_time": 60.66464094724506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1093.0,
|
|
"completions/max_terminated_length": 1093.0,
|
|
"completions/mean_length": 664.828125,
|
|
"completions/mean_terminated_length": 664.828125,
|
|
"completions/min_length": 256.0,
|
|
"completions/min_terminated_length": 256.0,
|
|
"entropy": 1.1280731745064259,
|
|
"epoch": 0.1621315192743764,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.00682494361535646,
|
|
"learning_rate": 8.384353741496598e-07,
|
|
"loss": -0.0094,
|
|
"num_tokens": 17013465.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 286,
|
|
"step_time": 40.771853673271835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1941.0,
|
|
"completions/max_terminated_length": 1941.0,
|
|
"completions/mean_length": 774.5625,
|
|
"completions/mean_terminated_length": 774.5625,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.9306526109576225,
|
|
"epoch": 0.1626984126984127,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.014398715313291177,
|
|
"learning_rate": 8.378684807256235e-07,
|
|
"loss": 0.0087,
|
|
"num_tokens": 17073797.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6167611479759216,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 287,
|
|
"step_time": 71.92694858182222
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1740.0,
|
|
"completions/max_terminated_length": 1740.0,
|
|
"completions/mean_length": 921.953125,
|
|
"completions/mean_terminated_length": 921.953125,
|
|
"completions/min_length": 394.0,
|
|
"completions/min_terminated_length": 394.0,
|
|
"entropy": 0.6797251738607883,
|
|
"epoch": 0.16326530612244897,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.004947350826114416,
|
|
"learning_rate": 8.373015873015873e-07,
|
|
"loss": -0.0098,
|
|
"num_tokens": 17144482.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 288,
|
|
"step_time": 64.15770970564336
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1693.0,
|
|
"completions/max_terminated_length": 1693.0,
|
|
"completions/mean_length": 753.03125,
|
|
"completions/mean_terminated_length": 753.03125,
|
|
"completions/min_length": 410.0,
|
|
"completions/min_terminated_length": 410.0,
|
|
"entropy": 0.9771784581243992,
|
|
"epoch": 0.16383219954648526,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.007738123604212888,
|
|
"learning_rate": 8.36734693877551e-07,
|
|
"loss": -0.0303,
|
|
"num_tokens": 17203300.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 289,
|
|
"step_time": 61.81910063419491
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1395.0,
|
|
"completions/max_terminated_length": 1395.0,
|
|
"completions/mean_length": 695.84375,
|
|
"completions/mean_terminated_length": 695.84375,
|
|
"completions/min_length": 390.0,
|
|
"completions/min_terminated_length": 390.0,
|
|
"entropy": 0.8843433819711208,
|
|
"epoch": 0.16439909297052155,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.007000370969763026,
|
|
"learning_rate": 8.361678004535147e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 17258106.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 290,
|
|
"step_time": 50.315598805435
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 687.65625,
|
|
"completions/mean_terminated_length": 687.65625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.8731617815792561,
|
|
"epoch": 0.1649659863945578,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.263671875,
|
|
"kl": 0.009098797789192759,
|
|
"learning_rate": 8.356009070294783e-07,
|
|
"loss": 0.0123,
|
|
"num_tokens": 17313788.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 291,
|
|
"step_time": 51.22578653600067
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1331.0,
|
|
"completions/max_terminated_length": 1331.0,
|
|
"completions/mean_length": 705.53125,
|
|
"completions/mean_terminated_length": 705.53125,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 1.038501761853695,
|
|
"epoch": 0.1655328798185941,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.01143992722791154,
|
|
"learning_rate": 8.350340136054421e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 17373430.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3435921370983124,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 292,
|
|
"step_time": 50.04920950811356
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1142.0,
|
|
"completions/max_terminated_length": 1142.0,
|
|
"completions/mean_length": 639.75,
|
|
"completions/mean_terminated_length": 639.75,
|
|
"completions/min_length": 199.0,
|
|
"completions/min_terminated_length": 199.0,
|
|
"entropy": 0.9801824130117893,
|
|
"epoch": 0.1660997732426304,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.006710244415444322,
|
|
"learning_rate": 8.344671201814058e-07,
|
|
"loss": -0.0121,
|
|
"num_tokens": 17425430.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 293,
|
|
"step_time": 42.188250828534365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1590.0,
|
|
"completions/max_terminated_length": 1590.0,
|
|
"completions/mean_length": 809.890625,
|
|
"completions/mean_terminated_length": 809.890625,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 0.9223752617835999,
|
|
"epoch": 0.16666666666666666,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.396484375,
|
|
"kl": 0.006054805249732453,
|
|
"learning_rate": 8.339002267573696e-07,
|
|
"loss": 0.0244,
|
|
"num_tokens": 17490111.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7339553236961365,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 294,
|
|
"step_time": 58.92305904533714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2058.0,
|
|
"completions/max_terminated_length": 2058.0,
|
|
"completions/mean_length": 762.875,
|
|
"completions/mean_terminated_length": 762.875,
|
|
"completions/min_length": 221.0,
|
|
"completions/min_terminated_length": 221.0,
|
|
"entropy": 0.7460720557719469,
|
|
"epoch": 0.16723356009070295,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.416015625,
|
|
"kl": 0.004036792801343836,
|
|
"learning_rate": 8.333333333333333e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 17549871.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 295,
|
|
"step_time": 75.95836392231286
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1319.0,
|
|
"completions/max_terminated_length": 1319.0,
|
|
"completions/mean_length": 680.90625,
|
|
"completions/mean_terminated_length": 680.90625,
|
|
"completions/min_length": 289.0,
|
|
"completions/min_terminated_length": 289.0,
|
|
"entropy": 0.7744119726121426,
|
|
"epoch": 0.16780045351473924,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.64453125,
|
|
"kl": 0.004806325436220504,
|
|
"learning_rate": 8.327664399092971e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 17603681.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5422958731651306,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 296,
|
|
"step_time": 48.54740091320127
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1365.0,
|
|
"completions/max_terminated_length": 1365.0,
|
|
"completions/mean_length": 772.546875,
|
|
"completions/mean_terminated_length": 772.546875,
|
|
"completions/min_length": 322.0,
|
|
"completions/min_terminated_length": 322.0,
|
|
"entropy": 0.7225805297493935,
|
|
"epoch": 0.1683673469387755,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.004341167637903709,
|
|
"learning_rate": 8.321995464852607e-07,
|
|
"loss": 0.0546,
|
|
"num_tokens": 17663668.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 297,
|
|
"step_time": 49.85294338222593
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1479.0,
|
|
"completions/mean_length": 856.109375,
|
|
"completions/mean_terminated_length": 804.6825561523438,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.819762421771884,
|
|
"epoch": 0.1689342403628118,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.006123719067545608,
|
|
"learning_rate": 8.316326530612245e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 17728627.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 298,
|
|
"step_time": 173.51606942806393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/max_terminated_length": 1389.0,
|
|
"completions/mean_length": 711.5,
|
|
"completions/mean_terminated_length": 711.5,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 1.0076992735266685,
|
|
"epoch": 0.16950113378684808,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.42578125,
|
|
"kl": 0.015792973979841918,
|
|
"learning_rate": 8.310657596371881e-07,
|
|
"loss": -0.0079,
|
|
"num_tokens": 17785603.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 299,
|
|
"step_time": 50.72464693058282
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1502.0,
|
|
"completions/max_terminated_length": 1502.0,
|
|
"completions/mean_length": 806.609375,
|
|
"completions/mean_terminated_length": 806.609375,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.6680117640644312,
|
|
"epoch": 0.17006802721088435,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.00916920488816686,
|
|
"learning_rate": 8.304988662131519e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 17849186.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 300,
|
|
"step_time": 54.90682506840676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1255.0,
|
|
"completions/max_terminated_length": 1255.0,
|
|
"completions/mean_length": 699.234375,
|
|
"completions/mean_terminated_length": 699.234375,
|
|
"completions/min_length": 30.0,
|
|
"completions/min_terminated_length": 30.0,
|
|
"entropy": 0.8960948511958122,
|
|
"epoch": 0.17063492063492064,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.62109375,
|
|
"kl": 0.009604600025340915,
|
|
"learning_rate": 8.299319727891156e-07,
|
|
"loss": -0.0302,
|
|
"num_tokens": 17905065.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 301,
|
|
"step_time": 45.49867108184844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1305.0,
|
|
"completions/max_terminated_length": 1305.0,
|
|
"completions/mean_length": 689.890625,
|
|
"completions/mean_terminated_length": 689.890625,
|
|
"completions/min_length": 26.0,
|
|
"completions/min_terminated_length": 26.0,
|
|
"entropy": 0.8179099150002003,
|
|
"epoch": 0.1712018140589569,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.01156878836627584,
|
|
"learning_rate": 8.293650793650794e-07,
|
|
"loss": 0.0077,
|
|
"num_tokens": 17960138.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.2083333283662796,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.109375,
|
|
"rewards/format_reward_func/std": 0.2083333432674408,
|
|
"step": 302,
|
|
"step_time": 46.90971658192575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1887.0,
|
|
"completions/max_terminated_length": 1887.0,
|
|
"completions/mean_length": 793.046875,
|
|
"completions/mean_terminated_length": 793.046875,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.8214219212532043,
|
|
"epoch": 0.1717687074829932,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.011117888643639162,
|
|
"learning_rate": 8.28798185941043e-07,
|
|
"loss": 0.0095,
|
|
"num_tokens": 18021213.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 303,
|
|
"step_time": 67.9431212414056
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1247.0,
|
|
"completions/max_terminated_length": 1247.0,
|
|
"completions/mean_length": 699.15625,
|
|
"completions/mean_terminated_length": 699.15625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.8105713315308094,
|
|
"epoch": 0.17233560090702948,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.003976978739956394,
|
|
"learning_rate": 8.282312925170068e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 18074143.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 304,
|
|
"step_time": 44.68646402191371
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1519.0,
|
|
"completions/max_terminated_length": 1519.0,
|
|
"completions/mean_length": 770.28125,
|
|
"completions/mean_terminated_length": 770.28125,
|
|
"completions/min_length": 243.0,
|
|
"completions/min_terminated_length": 243.0,
|
|
"entropy": 0.8100260365754366,
|
|
"epoch": 0.17290249433106575,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.005946670476987492,
|
|
"learning_rate": 8.276643990929705e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 18134369.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 305,
|
|
"step_time": 55.88502377551049
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 792.5625,
|
|
"completions/mean_terminated_length": 792.5625,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.6959674190729856,
|
|
"epoch": 0.17346938775510204,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.44921875,
|
|
"kl": 0.010840905888471752,
|
|
"learning_rate": 8.270975056689343e-07,
|
|
"loss": 0.0076,
|
|
"num_tokens": 18197653.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 306,
|
|
"step_time": 50.81538745574653
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1070.0,
|
|
"completions/max_terminated_length": 1070.0,
|
|
"completions/mean_length": 738.96875,
|
|
"completions/mean_terminated_length": 738.96875,
|
|
"completions/min_length": 238.0,
|
|
"completions/min_terminated_length": 238.0,
|
|
"entropy": 0.9731320925056934,
|
|
"epoch": 0.17403628117913833,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.005864134756848216,
|
|
"learning_rate": 8.26530612244898e-07,
|
|
"loss": -0.0048,
|
|
"num_tokens": 18255139.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 307,
|
|
"step_time": 38.74946460779756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1482.0,
|
|
"completions/max_terminated_length": 1482.0,
|
|
"completions/mean_length": 744.671875,
|
|
"completions/mean_terminated_length": 744.671875,
|
|
"completions/min_length": 293.0,
|
|
"completions/min_terminated_length": 293.0,
|
|
"entropy": 0.7935100309550762,
|
|
"epoch": 0.1746031746031746,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.0056961430091178045,
|
|
"learning_rate": 8.259637188208616e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 18319262.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7811011672019958,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 308,
|
|
"step_time": 55.75887096114457
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1193.0,
|
|
"completions/max_terminated_length": 1193.0,
|
|
"completions/mean_length": 667.40625,
|
|
"completions/mean_terminated_length": 667.40625,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 0.9020283985882998,
|
|
"epoch": 0.17517006802721088,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.03261678959825076,
|
|
"learning_rate": 8.253968253968253e-07,
|
|
"loss": 0.0068,
|
|
"num_tokens": 18373248.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 309,
|
|
"step_time": 42.62671375833452
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1227.0,
|
|
"completions/max_terminated_length": 1227.0,
|
|
"completions/mean_length": 694.5,
|
|
"completions/mean_terminated_length": 694.5,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 0.8163422644138336,
|
|
"epoch": 0.17573696145124718,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.69921875,
|
|
"kl": 0.007384991717117373,
|
|
"learning_rate": 8.24829931972789e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 18428016.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7559289336204529,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 310,
|
|
"step_time": 44.21994534600526
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1493.0,
|
|
"completions/max_terminated_length": 1493.0,
|
|
"completions/mean_length": 719.0,
|
|
"completions/mean_terminated_length": 719.0,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.9615583010017872,
|
|
"epoch": 0.17630385487528344,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58984375,
|
|
"kl": 0.011452683342213277,
|
|
"learning_rate": 8.242630385487528e-07,
|
|
"loss": 0.0143,
|
|
"num_tokens": 18484128.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.665922224521637,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 311,
|
|
"step_time": 53.601689458824694
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1279.0,
|
|
"completions/max_terminated_length": 1279.0,
|
|
"completions/mean_length": 747.96875,
|
|
"completions/mean_terminated_length": 747.96875,
|
|
"completions/min_length": 205.0,
|
|
"completions/min_terminated_length": 205.0,
|
|
"entropy": 0.8322487156838179,
|
|
"epoch": 0.17687074829931973,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.008456386131001636,
|
|
"learning_rate": 8.236961451247165e-07,
|
|
"loss": -0.0169,
|
|
"num_tokens": 18542350.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8296047449111938,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 312,
|
|
"step_time": 45.08718290459365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1363.0,
|
|
"completions/max_terminated_length": 1363.0,
|
|
"completions/mean_length": 692.75,
|
|
"completions/mean_terminated_length": 692.75,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 0.8413090296089649,
|
|
"epoch": 0.17743764172335602,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.007160043605836108,
|
|
"learning_rate": 8.231292517006803e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 18596598.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 313,
|
|
"step_time": 48.35842065606266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1331.0,
|
|
"completions/max_terminated_length": 1331.0,
|
|
"completions/mean_length": 770.21875,
|
|
"completions/mean_terminated_length": 770.21875,
|
|
"completions/min_length": 398.0,
|
|
"completions/min_terminated_length": 398.0,
|
|
"entropy": 0.8696554526686668,
|
|
"epoch": 0.17800453514739228,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.00865155813517049,
|
|
"learning_rate": 8.22562358276644e-07,
|
|
"loss": 0.025,
|
|
"num_tokens": 18655588.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6419816017150879,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 314,
|
|
"step_time": 46.93548165168613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 971.0,
|
|
"completions/max_terminated_length": 971.0,
|
|
"completions/mean_length": 614.375,
|
|
"completions/mean_terminated_length": 614.375,
|
|
"completions/min_length": 294.0,
|
|
"completions/min_terminated_length": 294.0,
|
|
"entropy": 0.9113731905817986,
|
|
"epoch": 0.17857142857142858,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6328125,
|
|
"kl": 0.0057736381713766605,
|
|
"learning_rate": 8.219954648526076e-07,
|
|
"loss": -0.0173,
|
|
"num_tokens": 18705884.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 315,
|
|
"step_time": 35.24758036993444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1689.0,
|
|
"completions/max_terminated_length": 1689.0,
|
|
"completions/mean_length": 785.59375,
|
|
"completions/mean_terminated_length": 785.59375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.7394520230591297,
|
|
"epoch": 0.17913832199546487,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.32421875,
|
|
"kl": 0.004618886479875073,
|
|
"learning_rate": 8.214285714285713e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 18767962.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.41277948021888733,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 316,
|
|
"step_time": 60.534888616763055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1130.0,
|
|
"completions/max_terminated_length": 1130.0,
|
|
"completions/mean_length": 690.171875,
|
|
"completions/mean_terminated_length": 690.171875,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.9527055807411671,
|
|
"epoch": 0.17970521541950113,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.007261134553118609,
|
|
"learning_rate": 8.208616780045351e-07,
|
|
"loss": -0.0121,
|
|
"num_tokens": 18822853.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 317,
|
|
"step_time": 39.881969602778554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1123.0,
|
|
"completions/max_terminated_length": 1123.0,
|
|
"completions/mean_length": 664.859375,
|
|
"completions/mean_terminated_length": 664.859375,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 0.765125572681427,
|
|
"epoch": 0.18027210884353742,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6796875,
|
|
"kl": 0.005805548498756252,
|
|
"learning_rate": 8.202947845804988e-07,
|
|
"loss": -0.0292,
|
|
"num_tokens": 18876444.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 318,
|
|
"step_time": 40.361245087347925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1142.0,
|
|
"completions/max_terminated_length": 1142.0,
|
|
"completions/mean_length": 643.703125,
|
|
"completions/mean_terminated_length": 643.703125,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 1.0180951990187168,
|
|
"epoch": 0.18083900226757368,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55078125,
|
|
"kl": 0.012444952561054379,
|
|
"learning_rate": 8.197278911564626e-07,
|
|
"loss": 0.0094,
|
|
"num_tokens": 18927193.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 319,
|
|
"step_time": 39.848433456383646
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1394.0,
|
|
"completions/max_terminated_length": 1394.0,
|
|
"completions/mean_length": 742.8125,
|
|
"completions/mean_terminated_length": 742.8125,
|
|
"completions/min_length": 411.0,
|
|
"completions/min_terminated_length": 411.0,
|
|
"entropy": 0.879846004769206,
|
|
"epoch": 0.18140589569160998,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.006181372940773144,
|
|
"learning_rate": 8.191609977324263e-07,
|
|
"loss": -0.0046,
|
|
"num_tokens": 18984853.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.4647814631462097,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.109375,
|
|
"rewards/format_reward_func/std": 0.2083333432674408,
|
|
"step": 320,
|
|
"step_time": 48.91933018993586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1921.0,
|
|
"completions/max_terminated_length": 1921.0,
|
|
"completions/mean_length": 762.21875,
|
|
"completions/mean_terminated_length": 762.21875,
|
|
"completions/min_length": 347.0,
|
|
"completions/min_terminated_length": 347.0,
|
|
"entropy": 0.671691881492734,
|
|
"epoch": 0.18197278911564627,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.007276202712091617,
|
|
"learning_rate": 8.1859410430839e-07,
|
|
"loss": 0.0197,
|
|
"num_tokens": 19044075.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 321,
|
|
"step_time": 67.99173118919134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1506.0,
|
|
"completions/max_terminated_length": 1506.0,
|
|
"completions/mean_length": 685.0625,
|
|
"completions/mean_terminated_length": 685.0625,
|
|
"completions/min_length": 211.0,
|
|
"completions/min_terminated_length": 211.0,
|
|
"entropy": 0.7449827585369349,
|
|
"epoch": 0.18253968253968253,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.00876174186123535,
|
|
"learning_rate": 8.180272108843537e-07,
|
|
"loss": -0.0183,
|
|
"num_tokens": 19098599.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5422958731651306,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 322,
|
|
"step_time": 53.3912045750767
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1408.0,
|
|
"completions/max_terminated_length": 1408.0,
|
|
"completions/mean_length": 853.375,
|
|
"completions/mean_terminated_length": 853.375,
|
|
"completions/min_length": 427.0,
|
|
"completions/min_terminated_length": 427.0,
|
|
"entropy": 0.6776500511914492,
|
|
"epoch": 0.18310657596371882,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.004744594931253232,
|
|
"learning_rate": 8.174603174603174e-07,
|
|
"loss": 0.0253,
|
|
"num_tokens": 19162807.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 323,
|
|
"step_time": 49.32134121377021
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1729.0,
|
|
"completions/max_terminated_length": 1729.0,
|
|
"completions/mean_length": 770.234375,
|
|
"completions/mean_terminated_length": 770.234375,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 0.8353247307240963,
|
|
"epoch": 0.1836734693877551,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.005294523012707941,
|
|
"learning_rate": 8.168934240362811e-07,
|
|
"loss": -0.034,
|
|
"num_tokens": 19221598.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6666666865348816,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 324,
|
|
"step_time": 60.75746985152364
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1198.0,
|
|
"completions/max_terminated_length": 1198.0,
|
|
"completions/mean_length": 710.0625,
|
|
"completions/mean_terminated_length": 710.0625,
|
|
"completions/min_length": 231.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"entropy": 0.8704922962933779,
|
|
"epoch": 0.18424036281179137,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6640625,
|
|
"kl": 0.006477083035861142,
|
|
"learning_rate": 8.163265306122449e-07,
|
|
"loss": -0.0105,
|
|
"num_tokens": 19278218.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 325,
|
|
"step_time": 42.49967502243817
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1223.0,
|
|
"completions/max_terminated_length": 1223.0,
|
|
"completions/mean_length": 693.765625,
|
|
"completions/mean_terminated_length": 693.765625,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.7533231135457754,
|
|
"epoch": 0.18480725623582767,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.014457460885751061,
|
|
"learning_rate": 8.157596371882086e-07,
|
|
"loss": 0.0182,
|
|
"num_tokens": 19333523.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 326,
|
|
"step_time": 42.95775222312659
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1570.0,
|
|
"completions/max_terminated_length": 1570.0,
|
|
"completions/mean_length": 838.71875,
|
|
"completions/mean_terminated_length": 838.71875,
|
|
"completions/min_length": 260.0,
|
|
"completions/min_terminated_length": 260.0,
|
|
"entropy": 0.6676177363842726,
|
|
"epoch": 0.18537414965986396,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.373046875,
|
|
"kl": 0.006262856913963333,
|
|
"learning_rate": 8.151927437641724e-07,
|
|
"loss": 0.0079,
|
|
"num_tokens": 19398801.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 327,
|
|
"step_time": 56.06219778954983
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1752.0,
|
|
"completions/max_terminated_length": 1752.0,
|
|
"completions/mean_length": 826.015625,
|
|
"completions/mean_terminated_length": 826.015625,
|
|
"completions/min_length": 434.0,
|
|
"completions/min_terminated_length": 434.0,
|
|
"entropy": 0.8412331230938435,
|
|
"epoch": 0.18594104308390022,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.010855631437152624,
|
|
"learning_rate": 8.14625850340136e-07,
|
|
"loss": 0.0193,
|
|
"num_tokens": 19461722.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 328,
|
|
"step_time": 61.37109683454037
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1725.0,
|
|
"completions/max_terminated_length": 1725.0,
|
|
"completions/mean_length": 770.890625,
|
|
"completions/mean_terminated_length": 770.890625,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.8527063429355621,
|
|
"epoch": 0.1865079365079365,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.007462741399649531,
|
|
"learning_rate": 8.140589569160997e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 19522291.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 329,
|
|
"step_time": 61.21692004892975
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1513.0,
|
|
"completions/max_terminated_length": 1513.0,
|
|
"completions/mean_length": 742.6875,
|
|
"completions/mean_terminated_length": 742.6875,
|
|
"completions/min_length": 46.0,
|
|
"completions/min_terminated_length": 46.0,
|
|
"entropy": 0.8760765083134174,
|
|
"epoch": 0.1870748299319728,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.007629668689332902,
|
|
"learning_rate": 8.134920634920635e-07,
|
|
"loss": 0.043,
|
|
"num_tokens": 19579751.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 330,
|
|
"step_time": 52.7058973480016
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1631.0,
|
|
"completions/max_terminated_length": 1631.0,
|
|
"completions/mean_length": 683.75,
|
|
"completions/mean_terminated_length": 683.75,
|
|
"completions/min_length": 121.0,
|
|
"completions/min_terminated_length": 121.0,
|
|
"entropy": 1.0992595627903938,
|
|
"epoch": 0.18764172335600907,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.008189230851712637,
|
|
"learning_rate": 8.129251700680271e-07,
|
|
"loss": -0.0046,
|
|
"num_tokens": 19633055.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 331,
|
|
"step_time": 58.39010221231729
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1540.0,
|
|
"completions/max_terminated_length": 1540.0,
|
|
"completions/mean_length": 854.65625,
|
|
"completions/mean_terminated_length": 854.65625,
|
|
"completions/min_length": 236.0,
|
|
"completions/min_terminated_length": 236.0,
|
|
"entropy": 0.8076948560774326,
|
|
"epoch": 0.18820861678004536,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2119140625,
|
|
"kl": 0.004547693082713522,
|
|
"learning_rate": 8.123582766439909e-07,
|
|
"loss": 0.0008,
|
|
"num_tokens": 19701201.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.32732683420181274,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 332,
|
|
"step_time": 54.694669464603066
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1356.0,
|
|
"completions/max_terminated_length": 1356.0,
|
|
"completions/mean_length": 837.5625,
|
|
"completions/mean_terminated_length": 837.5625,
|
|
"completions/min_length": 192.0,
|
|
"completions/min_terminated_length": 192.0,
|
|
"entropy": 0.7439503911882639,
|
|
"epoch": 0.18877551020408162,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.35546875,
|
|
"kl": 0.007064882040140219,
|
|
"learning_rate": 8.117913832199546e-07,
|
|
"loss": 0.0165,
|
|
"num_tokens": 19765029.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 333,
|
|
"step_time": 47.44372510071844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1374.0,
|
|
"completions/max_terminated_length": 1374.0,
|
|
"completions/mean_length": 748.5625,
|
|
"completions/mean_terminated_length": 748.5625,
|
|
"completions/min_length": 287.0,
|
|
"completions/min_terminated_length": 287.0,
|
|
"entropy": 0.8740064445883036,
|
|
"epoch": 0.1893424036281179,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3984375,
|
|
"kl": 0.012891482037957758,
|
|
"learning_rate": 8.112244897959183e-07,
|
|
"loss": 0.0077,
|
|
"num_tokens": 19822689.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.7126966714859009,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 334,
|
|
"step_time": 47.68842428550124
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1891.0,
|
|
"completions/max_terminated_length": 1891.0,
|
|
"completions/mean_length": 837.1875,
|
|
"completions/mean_terminated_length": 837.1875,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.655430793762207,
|
|
"epoch": 0.1899092970521542,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.006139033401268534,
|
|
"learning_rate": 8.10657596371882e-07,
|
|
"loss": 0.0307,
|
|
"num_tokens": 19887005.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.789156436920166,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 335,
|
|
"step_time": 65.92373467236757
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1099.0,
|
|
"completions/max_terminated_length": 1099.0,
|
|
"completions/mean_length": 646.640625,
|
|
"completions/mean_terminated_length": 646.640625,
|
|
"completions/min_length": 183.0,
|
|
"completions/min_terminated_length": 183.0,
|
|
"entropy": 0.8803813941776752,
|
|
"epoch": 0.19047619047619047,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.013101798918796703,
|
|
"learning_rate": 8.100907029478458e-07,
|
|
"loss": 0.0343,
|
|
"num_tokens": 19938014.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 336,
|
|
"step_time": 38.181463305838406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1683.0,
|
|
"completions/max_terminated_length": 1683.0,
|
|
"completions/mean_length": 890.3125,
|
|
"completions/mean_terminated_length": 890.3125,
|
|
"completions/min_length": 439.0,
|
|
"completions/min_terminated_length": 439.0,
|
|
"entropy": 0.6142123676836491,
|
|
"epoch": 0.19104308390022676,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.009792614495381713,
|
|
"learning_rate": 8.095238095238095e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 20005442.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 337,
|
|
"step_time": 59.30274276807904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1184.0,
|
|
"completions/max_terminated_length": 1184.0,
|
|
"completions/mean_length": 712.53125,
|
|
"completions/mean_terminated_length": 712.53125,
|
|
"completions/min_length": 156.0,
|
|
"completions/min_terminated_length": 156.0,
|
|
"entropy": 0.7328694984316826,
|
|
"epoch": 0.19160997732426305,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.310546875,
|
|
"kl": 0.030268765331129543,
|
|
"learning_rate": 8.089569160997733e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 20063588.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 338,
|
|
"step_time": 42.308118044398725
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1431.0,
|
|
"completions/max_terminated_length": 1431.0,
|
|
"completions/mean_length": 682.65625,
|
|
"completions/mean_terminated_length": 682.65625,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.9973875358700752,
|
|
"epoch": 0.1921768707482993,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.009495670878095552,
|
|
"learning_rate": 8.083900226757369e-07,
|
|
"loss": -0.0035,
|
|
"num_tokens": 20119038.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.7111289501190186,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 339,
|
|
"step_time": 51.06384574435651
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1514.0,
|
|
"completions/max_terminated_length": 1514.0,
|
|
"completions/mean_length": 837.703125,
|
|
"completions/mean_terminated_length": 837.703125,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.7334924042224884,
|
|
"epoch": 0.1927437641723356,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"kl": 0.006166985287563875,
|
|
"learning_rate": 8.078231292517006e-07,
|
|
"loss": -0.0146,
|
|
"num_tokens": 20183195.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.46922925114631653,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 340,
|
|
"step_time": 53.17411183193326
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2742.0,
|
|
"completions/max_terminated_length": 2742.0,
|
|
"completions/mean_length": 880.5,
|
|
"completions/mean_terminated_length": 880.5,
|
|
"completions/min_length": 399.0,
|
|
"completions/min_terminated_length": 399.0,
|
|
"entropy": 1.0167470518499613,
|
|
"epoch": 0.1933106575963719,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.01515770039986819,
|
|
"learning_rate": 8.072562358276643e-07,
|
|
"loss": -0.024,
|
|
"num_tokens": 20252307.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 341,
|
|
"step_time": 102.54252031445503
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/max_terminated_length": 1389.0,
|
|
"completions/mean_length": 827.375,
|
|
"completions/mean_terminated_length": 827.375,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.7021002303808928,
|
|
"epoch": 0.19387755102040816,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.00620406138477847,
|
|
"learning_rate": 8.066893424036281e-07,
|
|
"loss": 0.0052,
|
|
"num_tokens": 20315067.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 342,
|
|
"step_time": 48.8435224480927
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1466.0,
|
|
"completions/max_terminated_length": 1466.0,
|
|
"completions/mean_length": 900.28125,
|
|
"completions/mean_terminated_length": 900.28125,
|
|
"completions/min_length": 565.0,
|
|
"completions/min_terminated_length": 565.0,
|
|
"entropy": 0.6297467797994614,
|
|
"epoch": 0.19444444444444445,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.008732411442906596,
|
|
"learning_rate": 8.061224489795918e-07,
|
|
"loss": -0.0083,
|
|
"num_tokens": 20383693.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 343,
|
|
"step_time": 51.75457154493779
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1693.0,
|
|
"completions/max_terminated_length": 1693.0,
|
|
"completions/mean_length": 737.515625,
|
|
"completions/mean_terminated_length": 737.515625,
|
|
"completions/min_length": 259.0,
|
|
"completions/min_terminated_length": 259.0,
|
|
"entropy": 0.6419767197221518,
|
|
"epoch": 0.19501133786848074,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.005702765440219082,
|
|
"learning_rate": 8.055555555555556e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 20440726.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.728528618812561,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 344,
|
|
"step_time": 59.728486637584865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1250.0,
|
|
"completions/max_terminated_length": 1250.0,
|
|
"completions/mean_length": 752.296875,
|
|
"completions/mean_terminated_length": 752.296875,
|
|
"completions/min_length": 435.0,
|
|
"completions/min_terminated_length": 435.0,
|
|
"entropy": 0.9790338799357414,
|
|
"epoch": 0.195578231292517,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.009619498610845767,
|
|
"learning_rate": 8.049886621315193e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 20501713.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 345,
|
|
"step_time": 43.7907804697752
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1285.0,
|
|
"completions/max_terminated_length": 1285.0,
|
|
"completions/mean_length": 673.359375,
|
|
"completions/mean_terminated_length": 673.359375,
|
|
"completions/min_length": 40.0,
|
|
"completions/min_terminated_length": 40.0,
|
|
"entropy": 0.9337127301841974,
|
|
"epoch": 0.1961451247165533,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.66796875,
|
|
"kl": 0.00934422598220408,
|
|
"learning_rate": 8.04421768707483e-07,
|
|
"loss": 0.0401,
|
|
"num_tokens": 20555952.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.8237544298171997,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 346,
|
|
"step_time": 45.38670400623232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2688.0,
|
|
"completions/max_terminated_length": 2688.0,
|
|
"completions/mean_length": 761.03125,
|
|
"completions/mean_terminated_length": 761.03125,
|
|
"completions/min_length": 292.0,
|
|
"completions/min_terminated_length": 292.0,
|
|
"entropy": 0.4654211699962616,
|
|
"epoch": 0.19671201814058956,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.0064781456894706935,
|
|
"learning_rate": 8.038548752834466e-07,
|
|
"loss": 0.0359,
|
|
"num_tokens": 20613442.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7194505929946899,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 347,
|
|
"step_time": 98.53148745372891
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1560.0,
|
|
"completions/max_terminated_length": 1560.0,
|
|
"completions/mean_length": 800.109375,
|
|
"completions/mean_terminated_length": 800.109375,
|
|
"completions/min_length": 465.0,
|
|
"completions/min_terminated_length": 465.0,
|
|
"entropy": 0.7911105696111917,
|
|
"epoch": 0.19727891156462585,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.007789595707436092,
|
|
"learning_rate": 8.032879818594103e-07,
|
|
"loss": -0.0085,
|
|
"num_tokens": 20674321.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 348,
|
|
"step_time": 54.41751306131482
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1034.0,
|
|
"completions/max_terminated_length": 1034.0,
|
|
"completions/mean_length": 695.703125,
|
|
"completions/mean_terminated_length": 695.703125,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 1.1095048561692238,
|
|
"epoch": 0.19784580498866214,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.012152556111686863,
|
|
"learning_rate": 8.027210884353741e-07,
|
|
"loss": 0.0108,
|
|
"num_tokens": 20730230.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6164092421531677,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 349,
|
|
"step_time": 36.84613507427275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1679.0,
|
|
"completions/max_terminated_length": 1679.0,
|
|
"completions/mean_length": 767.109375,
|
|
"completions/mean_terminated_length": 767.109375,
|
|
"completions/min_length": 305.0,
|
|
"completions/min_terminated_length": 305.0,
|
|
"entropy": 0.8563258647918701,
|
|
"epoch": 0.1984126984126984,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.369140625,
|
|
"kl": 0.011192118050530553,
|
|
"learning_rate": 8.021541950113378e-07,
|
|
"loss": 0.0244,
|
|
"num_tokens": 20794885.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 350,
|
|
"step_time": 63.66168717946857
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1726.0,
|
|
"completions/max_terminated_length": 1726.0,
|
|
"completions/mean_length": 766.4375,
|
|
"completions/mean_terminated_length": 766.4375,
|
|
"completions/min_length": 304.0,
|
|
"completions/min_terminated_length": 304.0,
|
|
"entropy": 0.808762526139617,
|
|
"epoch": 0.1989795918367347,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.63671875,
|
|
"kl": 0.004394636191136669,
|
|
"learning_rate": 8.015873015873016e-07,
|
|
"loss": -0.0018,
|
|
"num_tokens": 20855457.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 351,
|
|
"step_time": 61.878230896778405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1161.0,
|
|
"completions/max_terminated_length": 1161.0,
|
|
"completions/mean_length": 730.625,
|
|
"completions/mean_terminated_length": 730.625,
|
|
"completions/min_length": 369.0,
|
|
"completions/min_terminated_length": 369.0,
|
|
"entropy": 0.7505794577300549,
|
|
"epoch": 0.19954648526077098,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.0053577764047076926,
|
|
"learning_rate": 8.010204081632652e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 20912017.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 352,
|
|
"step_time": 40.289197850972414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1783.0,
|
|
"completions/max_terminated_length": 1783.0,
|
|
"completions/mean_length": 776.0625,
|
|
"completions/mean_terminated_length": 776.0625,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 1.0009315647184849,
|
|
"epoch": 0.20011337868480725,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.005335954439942725,
|
|
"learning_rate": 8.00453514739229e-07,
|
|
"loss": -0.0115,
|
|
"num_tokens": 20971821.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 353,
|
|
"step_time": 62.82513804361224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1211.0,
|
|
"completions/mean_length": 826.515625,
|
|
"completions/mean_terminated_length": 721.04833984375,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 0.6306920368224382,
|
|
"epoch": 0.20068027210884354,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.004599000429152511,
|
|
"learning_rate": 7.998866213151927e-07,
|
|
"loss": -0.0351,
|
|
"num_tokens": 21034734.0,
|
|
"reward": 0.6875,
|
|
"reward_std": 1.005934715270996,
|
|
"rewards/accuracy_reward_func/mean": 0.625,
|
|
"rewards/accuracy_reward_func/std": 0.934353232383728,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 354,
|
|
"step_time": 170.80954848509282
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1284.0,
|
|
"completions/max_terminated_length": 1284.0,
|
|
"completions/mean_length": 793.71875,
|
|
"completions/mean_terminated_length": 793.71875,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 0.9095571152865887,
|
|
"epoch": 0.20124716553287983,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.006502260177512653,
|
|
"learning_rate": 7.993197278911564e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 21094500.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 355,
|
|
"step_time": 45.50296873413026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1178.0,
|
|
"completions/max_terminated_length": 1178.0,
|
|
"completions/mean_length": 699.28125,
|
|
"completions/mean_terminated_length": 699.28125,
|
|
"completions/min_length": 253.0,
|
|
"completions/min_terminated_length": 253.0,
|
|
"entropy": 0.7517442852258682,
|
|
"epoch": 0.2018140589569161,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2265625,
|
|
"kl": 0.009225394751410931,
|
|
"learning_rate": 7.987528344671201e-07,
|
|
"loss": -0.0013,
|
|
"num_tokens": 21153798.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.5058560967445374,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 356,
|
|
"step_time": 43.04798846133053
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1961.0,
|
|
"completions/max_terminated_length": 1961.0,
|
|
"completions/mean_length": 805.625,
|
|
"completions/mean_terminated_length": 805.625,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.7604002766311169,
|
|
"epoch": 0.20238095238095238,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.006553398416144773,
|
|
"learning_rate": 7.981859410430839e-07,
|
|
"loss": 0.0395,
|
|
"num_tokens": 21214686.0,
|
|
"reward": 0.4765625,
|
|
"reward_std": 0.8424721956253052,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 357,
|
|
"step_time": 68.64221620652825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1548.0,
|
|
"completions/max_terminated_length": 1548.0,
|
|
"completions/mean_length": 774.65625,
|
|
"completions/mean_terminated_length": 774.65625,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 0.6257712636142969,
|
|
"epoch": 0.20294784580498867,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.009716352680698037,
|
|
"learning_rate": 7.976190476190476e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 21275408.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 358,
|
|
"step_time": 54.75614361651242
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1500.0,
|
|
"completions/max_terminated_length": 1500.0,
|
|
"completions/mean_length": 647.3125,
|
|
"completions/mean_terminated_length": 647.3125,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.9415576122701168,
|
|
"epoch": 0.20351473922902494,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.58984375,
|
|
"kl": 0.008676898840349168,
|
|
"learning_rate": 7.970521541950113e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 21327556.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 359,
|
|
"step_time": 52.70056934468448
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1533.0,
|
|
"completions/max_terminated_length": 1533.0,
|
|
"completions/mean_length": 738.34375,
|
|
"completions/mean_terminated_length": 738.34375,
|
|
"completions/min_length": 267.0,
|
|
"completions/min_terminated_length": 267.0,
|
|
"entropy": 0.6640665493905544,
|
|
"epoch": 0.20408163265306123,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.0048632768739480525,
|
|
"learning_rate": 7.96485260770975e-07,
|
|
"loss": 0.0163,
|
|
"num_tokens": 21384250.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 360,
|
|
"step_time": 53.82986585609615
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1365.0,
|
|
"completions/max_terminated_length": 1365.0,
|
|
"completions/mean_length": 806.40625,
|
|
"completions/mean_terminated_length": 806.40625,
|
|
"completions/min_length": 438.0,
|
|
"completions/min_terminated_length": 438.0,
|
|
"entropy": 0.897387720644474,
|
|
"epoch": 0.2046485260770975,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.466796875,
|
|
"kl": 0.004769071820192039,
|
|
"learning_rate": 7.959183673469388e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 21446972.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 361,
|
|
"step_time": 49.43968720827252
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/max_terminated_length": 1389.0,
|
|
"completions/mean_length": 751.515625,
|
|
"completions/mean_terminated_length": 751.515625,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 0.9682449884712696,
|
|
"epoch": 0.20521541950113378,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.0074831047968473285,
|
|
"learning_rate": 7.953514739229025e-07,
|
|
"loss": 0.0206,
|
|
"num_tokens": 21504861.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 362,
|
|
"step_time": 48.7774790963158
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1300.0,
|
|
"completions/max_terminated_length": 1300.0,
|
|
"completions/mean_length": 673.171875,
|
|
"completions/mean_terminated_length": 673.171875,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.8318615090101957,
|
|
"epoch": 0.20578231292517007,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.006348504466586746,
|
|
"learning_rate": 7.947845804988662e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 21558000.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 363,
|
|
"step_time": 45.67016400489956
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1382.0,
|
|
"completions/max_terminated_length": 1382.0,
|
|
"completions/mean_length": 620.296875,
|
|
"completions/mean_terminated_length": 620.296875,
|
|
"completions/min_length": 175.0,
|
|
"completions/min_terminated_length": 175.0,
|
|
"entropy": 0.9658241774886847,
|
|
"epoch": 0.20634920634920634,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.019720278680324554,
|
|
"learning_rate": 7.942176870748299e-07,
|
|
"loss": -0.0023,
|
|
"num_tokens": 21610395.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 364,
|
|
"step_time": 49.3195380019024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1189.0,
|
|
"completions/max_terminated_length": 1189.0,
|
|
"completions/mean_length": 728.140625,
|
|
"completions/mean_terminated_length": 728.140625,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.876466816291213,
|
|
"epoch": 0.20691609977324263,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.671875,
|
|
"kl": 0.018591600528452545,
|
|
"learning_rate": 7.936507936507936e-07,
|
|
"loss": -0.0426,
|
|
"num_tokens": 21665964.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 365,
|
|
"step_time": 44.70258327387273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1513.0,
|
|
"completions/max_terminated_length": 1513.0,
|
|
"completions/mean_length": 734.28125,
|
|
"completions/mean_terminated_length": 734.28125,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.7373211551457644,
|
|
"epoch": 0.20748299319727892,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.03787548135733232,
|
|
"learning_rate": 7.930839002267573e-07,
|
|
"loss": 0.0167,
|
|
"num_tokens": 21725278.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 366,
|
|
"step_time": 55.486514929682016
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1356.0,
|
|
"completions/max_terminated_length": 1356.0,
|
|
"completions/mean_length": 826.609375,
|
|
"completions/mean_terminated_length": 826.609375,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.803842194378376,
|
|
"epoch": 0.20804988662131518,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.01093247470271308,
|
|
"learning_rate": 7.92517006802721e-07,
|
|
"loss": 0.0045,
|
|
"num_tokens": 21788645.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 367,
|
|
"step_time": 48.00471571832895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/max_terminated_length": 1389.0,
|
|
"completions/mean_length": 688.078125,
|
|
"completions/mean_terminated_length": 688.078125,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 1.0153057016432285,
|
|
"epoch": 0.20861678004535147,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.009997964443755336,
|
|
"learning_rate": 7.919501133786848e-07,
|
|
"loss": 0.0096,
|
|
"num_tokens": 21843770.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 368,
|
|
"step_time": 49.48769662436098
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1204.0,
|
|
"completions/max_terminated_length": 1204.0,
|
|
"completions/mean_length": 751.9375,
|
|
"completions/mean_terminated_length": 751.9375,
|
|
"completions/min_length": 419.0,
|
|
"completions/min_terminated_length": 419.0,
|
|
"entropy": 0.7951037138700485,
|
|
"epoch": 0.20918367346938777,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.006458746662246995,
|
|
"learning_rate": 7.913832199546485e-07,
|
|
"loss": -0.0013,
|
|
"num_tokens": 21902414.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 369,
|
|
"step_time": 42.71605859789997
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1887.0,
|
|
"completions/max_terminated_length": 1887.0,
|
|
"completions/mean_length": 770.671875,
|
|
"completions/mean_terminated_length": 770.671875,
|
|
"completions/min_length": 160.0,
|
|
"completions/min_terminated_length": 160.0,
|
|
"entropy": 0.8654769137501717,
|
|
"epoch": 0.20975056689342403,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.007016115239821374,
|
|
"learning_rate": 7.908163265306123e-07,
|
|
"loss": -0.0292,
|
|
"num_tokens": 21962993.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6529466509819031,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 370,
|
|
"step_time": 66.75023751053959
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1402.0,
|
|
"completions/max_terminated_length": 1402.0,
|
|
"completions/mean_length": 711.1875,
|
|
"completions/mean_terminated_length": 711.1875,
|
|
"completions/min_length": 363.0,
|
|
"completions/min_terminated_length": 363.0,
|
|
"entropy": 0.9247559905052185,
|
|
"epoch": 0.21031746031746032,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.01341819655499421,
|
|
"learning_rate": 7.902494331065758e-07,
|
|
"loss": -0.01,
|
|
"num_tokens": 22018997.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 371,
|
|
"step_time": 49.6273089479655
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1664.0,
|
|
"completions/max_terminated_length": 1664.0,
|
|
"completions/mean_length": 824.03125,
|
|
"completions/mean_terminated_length": 824.03125,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.782563166692853,
|
|
"epoch": 0.2108843537414966,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2060546875,
|
|
"kl": 0.008505658013746142,
|
|
"learning_rate": 7.896825396825396e-07,
|
|
"loss": 0.0049,
|
|
"num_tokens": 22080911.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6777355074882507,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 372,
|
|
"step_time": 58.256893769837916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1581.0,
|
|
"completions/max_terminated_length": 1581.0,
|
|
"completions/mean_length": 856.078125,
|
|
"completions/mean_terminated_length": 856.078125,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.7458044551312923,
|
|
"epoch": 0.21145124716553287,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.392578125,
|
|
"kl": 0.0071403429537895136,
|
|
"learning_rate": 7.891156462585033e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 22145620.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 373,
|
|
"step_time": 55.691850893199444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1012.0,
|
|
"completions/max_terminated_length": 1012.0,
|
|
"completions/mean_length": 675.375,
|
|
"completions/mean_terminated_length": 675.375,
|
|
"completions/min_length": 287.0,
|
|
"completions/min_terminated_length": 287.0,
|
|
"entropy": 1.13796192035079,
|
|
"epoch": 0.21201814058956917,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.0060508267633849755,
|
|
"learning_rate": 7.885487528344671e-07,
|
|
"loss": -0.0124,
|
|
"num_tokens": 22198964.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7319250702857971,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 374,
|
|
"step_time": 36.025930147618055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1204.0,
|
|
"completions/max_terminated_length": 1204.0,
|
|
"completions/mean_length": 728.03125,
|
|
"completions/mean_terminated_length": 728.03125,
|
|
"completions/min_length": 280.0,
|
|
"completions/min_terminated_length": 280.0,
|
|
"entropy": 0.7827387042343616,
|
|
"epoch": 0.21258503401360543,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.392578125,
|
|
"kl": 0.006068907270673662,
|
|
"learning_rate": 7.879818594104308e-07,
|
|
"loss": 0.0027,
|
|
"num_tokens": 22255678.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 375,
|
|
"step_time": 42.63181810081005
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1404.0,
|
|
"completions/max_terminated_length": 1404.0,
|
|
"completions/mean_length": 736.28125,
|
|
"completions/mean_terminated_length": 736.28125,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.6721552610397339,
|
|
"epoch": 0.21315192743764172,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6171875,
|
|
"kl": 0.005795677818241529,
|
|
"learning_rate": 7.874149659863946e-07,
|
|
"loss": 0.0195,
|
|
"num_tokens": 22313056.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.795534074306488,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 376,
|
|
"step_time": 49.154366455972195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1287.0,
|
|
"completions/max_terminated_length": 1287.0,
|
|
"completions/mean_length": 644.5,
|
|
"completions/mean_terminated_length": 644.5,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.6348938029259443,
|
|
"epoch": 0.213718820861678,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.006758220930350944,
|
|
"learning_rate": 7.868480725623582e-07,
|
|
"loss": 0.0116,
|
|
"num_tokens": 22362848.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 377,
|
|
"step_time": 45.09674393199384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2051.0,
|
|
"completions/max_terminated_length": 2051.0,
|
|
"completions/mean_length": 705.109375,
|
|
"completions/mean_terminated_length": 705.109375,
|
|
"completions/min_length": 342.0,
|
|
"completions/min_terminated_length": 342.0,
|
|
"entropy": 0.6979906857013702,
|
|
"epoch": 0.21428571428571427,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.020916220819344744,
|
|
"learning_rate": 7.86281179138322e-07,
|
|
"loss": 0.0318,
|
|
"num_tokens": 22418207.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8003905415534973,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 378,
|
|
"step_time": 72.91525127086788
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 740.421875,
|
|
"completions/mean_terminated_length": 740.421875,
|
|
"completions/min_length": 331.0,
|
|
"completions/min_terminated_length": 331.0,
|
|
"entropy": 1.0521539971232414,
|
|
"epoch": 0.21485260770975056,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.7421875,
|
|
"kl": 0.00853558856761083,
|
|
"learning_rate": 7.857142857142856e-07,
|
|
"loss": -0.0424,
|
|
"num_tokens": 22477218.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 379,
|
|
"step_time": 48.239318737760186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2350.0,
|
|
"completions/max_terminated_length": 2350.0,
|
|
"completions/mean_length": 785.5,
|
|
"completions/mean_terminated_length": 785.5,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.8634990677237511,
|
|
"epoch": 0.21541950113378686,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.007020699333224911,
|
|
"learning_rate": 7.851473922902494e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 22542362.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 380,
|
|
"step_time": 87.2783026220277
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1341.0,
|
|
"completions/max_terminated_length": 1341.0,
|
|
"completions/mean_length": 739.515625,
|
|
"completions/mean_terminated_length": 739.515625,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.6452418379485607,
|
|
"epoch": 0.21598639455782312,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.55078125,
|
|
"kl": 0.008970728755230084,
|
|
"learning_rate": 7.845804988662131e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 22599899.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 381,
|
|
"step_time": 47.75332001410425
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1873.0,
|
|
"completions/max_terminated_length": 1873.0,
|
|
"completions/mean_length": 758.328125,
|
|
"completions/mean_terminated_length": 758.328125,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 0.7167969793081284,
|
|
"epoch": 0.2165532879818594,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.014076261446462013,
|
|
"learning_rate": 7.840136054421769e-07,
|
|
"loss": -0.0125,
|
|
"num_tokens": 22658704.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 382,
|
|
"step_time": 65.95075592491776
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1037.0,
|
|
"completions/max_terminated_length": 1037.0,
|
|
"completions/mean_length": 569.5625,
|
|
"completions/mean_terminated_length": 569.5625,
|
|
"completions/min_length": 207.0,
|
|
"completions/min_terminated_length": 207.0,
|
|
"entropy": 0.9594349786639214,
|
|
"epoch": 0.2171201814058957,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.796875,
|
|
"kl": 0.009265722255804576,
|
|
"learning_rate": 7.834467120181405e-07,
|
|
"loss": -0.0062,
|
|
"num_tokens": 22706340.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 383,
|
|
"step_time": 37.67073441017419
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1525.0,
|
|
"completions/max_terminated_length": 1525.0,
|
|
"completions/mean_length": 876.65625,
|
|
"completions/mean_terminated_length": 876.65625,
|
|
"completions/min_length": 447.0,
|
|
"completions/min_terminated_length": 447.0,
|
|
"entropy": 0.887593649327755,
|
|
"epoch": 0.21768707482993196,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.296875,
|
|
"kl": 0.005941338909906335,
|
|
"learning_rate": 7.828798185941043e-07,
|
|
"loss": 0.006,
|
|
"num_tokens": 22775126.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5540592670440674,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 384,
|
|
"step_time": 54.38487575855106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1332.0,
|
|
"completions/max_terminated_length": 1332.0,
|
|
"completions/mean_length": 742.078125,
|
|
"completions/mean_terminated_length": 742.078125,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.9018192701041698,
|
|
"epoch": 0.21825396825396826,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.373046875,
|
|
"kl": 0.0055784515861887485,
|
|
"learning_rate": 7.82312925170068e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 22833043.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 385,
|
|
"step_time": 48.398129280656576
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 725.171875,
|
|
"completions/mean_terminated_length": 725.171875,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 0.9289520047605038,
|
|
"epoch": 0.21882086167800455,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.011218311949050985,
|
|
"learning_rate": 7.817460317460318e-07,
|
|
"loss": -0.0105,
|
|
"num_tokens": 22894374.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 386,
|
|
"step_time": 52.46727229747921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1949.0,
|
|
"completions/max_terminated_length": 1949.0,
|
|
"completions/mean_length": 664.46875,
|
|
"completions/mean_terminated_length": 664.46875,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 0.8146755807101727,
|
|
"epoch": 0.2193877551020408,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5390625,
|
|
"kl": 0.020864911552052945,
|
|
"learning_rate": 7.811791383219954e-07,
|
|
"loss": 0.021,
|
|
"num_tokens": 22949100.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.688941478729248,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 387,
|
|
"step_time": 71.70860512368381
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1321.0,
|
|
"completions/max_terminated_length": 1321.0,
|
|
"completions/mean_length": 740.78125,
|
|
"completions/mean_terminated_length": 740.78125,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.8137216381728649,
|
|
"epoch": 0.2199546485260771,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.007592617155751213,
|
|
"learning_rate": 7.806122448979591e-07,
|
|
"loss": -0.0027,
|
|
"num_tokens": 23006438.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 388,
|
|
"step_time": 46.02304187230766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1633.0,
|
|
"completions/max_terminated_length": 1633.0,
|
|
"completions/mean_length": 886.15625,
|
|
"completions/mean_terminated_length": 886.15625,
|
|
"completions/min_length": 493.0,
|
|
"completions/min_terminated_length": 493.0,
|
|
"entropy": 0.8164814542979002,
|
|
"epoch": 0.2205215419501134,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.412109375,
|
|
"kl": 0.004828404598811176,
|
|
"learning_rate": 7.800453514739229e-07,
|
|
"loss": -0.0077,
|
|
"num_tokens": 23075376.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6167611479759216,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 389,
|
|
"step_time": 57.176508142612875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1641.0,
|
|
"completions/max_terminated_length": 1641.0,
|
|
"completions/mean_length": 741.0,
|
|
"completions/mean_terminated_length": 741.0,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.8530206680297852,
|
|
"epoch": 0.22108843537414966,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.010848585545318201,
|
|
"learning_rate": 7.794784580498865e-07,
|
|
"loss": 0.0302,
|
|
"num_tokens": 23132752.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 390,
|
|
"step_time": 57.22498300112784
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1255.0,
|
|
"completions/max_terminated_length": 1255.0,
|
|
"completions/mean_length": 608.375,
|
|
"completions/mean_terminated_length": 608.375,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.8223625533282757,
|
|
"epoch": 0.22165532879818595,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.015227242663968354,
|
|
"learning_rate": 7.789115646258503e-07,
|
|
"loss": 0.0455,
|
|
"num_tokens": 23181792.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 391,
|
|
"step_time": 43.47986264619976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1616.0,
|
|
"completions/max_terminated_length": 1616.0,
|
|
"completions/mean_length": 780.921875,
|
|
"completions/mean_terminated_length": 780.921875,
|
|
"completions/min_length": 394.0,
|
|
"completions/min_terminated_length": 394.0,
|
|
"entropy": 0.7135511264204979,
|
|
"epoch": 0.2222222222222222,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.010050005716038868,
|
|
"learning_rate": 7.78344671201814e-07,
|
|
"loss": 0.0072,
|
|
"num_tokens": 23241811.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.8207381367683411,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 392,
|
|
"step_time": 56.368631350807846
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1227.0,
|
|
"completions/max_terminated_length": 1227.0,
|
|
"completions/mean_length": 735.453125,
|
|
"completions/mean_terminated_length": 735.453125,
|
|
"completions/min_length": 365.0,
|
|
"completions/min_terminated_length": 365.0,
|
|
"entropy": 0.9050820115953684,
|
|
"epoch": 0.2227891156462585,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.466796875,
|
|
"kl": 0.005837492266437039,
|
|
"learning_rate": 7.777777777777778e-07,
|
|
"loss": 0.0221,
|
|
"num_tokens": 23298648.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.6016985774040222,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 393,
|
|
"step_time": 43.485394665040076
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1343.0,
|
|
"completions/max_terminated_length": 1343.0,
|
|
"completions/mean_length": 807.890625,
|
|
"completions/mean_terminated_length": 807.890625,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.8196720741689205,
|
|
"epoch": 0.2233560090702948,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.009971795996534638,
|
|
"learning_rate": 7.772108843537415e-07,
|
|
"loss": 0.0204,
|
|
"num_tokens": 23361753.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 394,
|
|
"step_time": 47.62207393441349
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1605.0,
|
|
"completions/max_terminated_length": 1605.0,
|
|
"completions/mean_length": 822.5625,
|
|
"completions/mean_terminated_length": 822.5625,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.5809940975159407,
|
|
"epoch": 0.22392290249433106,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.004477146649151109,
|
|
"learning_rate": 7.766439909297052e-07,
|
|
"loss": 0.0224,
|
|
"num_tokens": 23425413.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 395,
|
|
"step_time": 56.65555914957076
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1438.0,
|
|
"completions/max_terminated_length": 1438.0,
|
|
"completions/mean_length": 715.125,
|
|
"completions/mean_terminated_length": 715.125,
|
|
"completions/min_length": 183.0,
|
|
"completions/min_terminated_length": 183.0,
|
|
"entropy": 0.9844364896416664,
|
|
"epoch": 0.22448979591836735,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6328125,
|
|
"kl": 0.007322670266148634,
|
|
"learning_rate": 7.760770975056688e-07,
|
|
"loss": 0.0234,
|
|
"num_tokens": 23482165.0,
|
|
"reward": 0.5078125,
|
|
"reward_std": 0.8796291947364807,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 396,
|
|
"step_time": 50.7606395855546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1679.0,
|
|
"completions/max_terminated_length": 1679.0,
|
|
"completions/mean_length": 638.4375,
|
|
"completions/mean_terminated_length": 638.4375,
|
|
"completions/min_length": 21.0,
|
|
"completions/min_terminated_length": 21.0,
|
|
"entropy": 0.8209603633731604,
|
|
"epoch": 0.22505668934240364,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.006006026364048012,
|
|
"learning_rate": 7.755102040816326e-07,
|
|
"loss": 0.0226,
|
|
"num_tokens": 23532073.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.7014281749725342,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 397,
|
|
"step_time": 58.50951211899519
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1314.0,
|
|
"completions/max_terminated_length": 1314.0,
|
|
"completions/mean_length": 745.4375,
|
|
"completions/mean_terminated_length": 745.4375,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 1.0559663251042366,
|
|
"epoch": 0.2256235827664399,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.011853849471663125,
|
|
"learning_rate": 7.749433106575963e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 23591981.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 398,
|
|
"step_time": 46.28473854716867
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/max_terminated_length": 1452.0,
|
|
"completions/mean_length": 732.6875,
|
|
"completions/mean_terminated_length": 732.6875,
|
|
"completions/min_length": 406.0,
|
|
"completions/min_terminated_length": 406.0,
|
|
"entropy": 0.9582856111228466,
|
|
"epoch": 0.2261904761904762,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.010223097779089585,
|
|
"learning_rate": 7.743764172335601e-07,
|
|
"loss": 0.002,
|
|
"num_tokens": 23648993.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 399,
|
|
"step_time": 50.92035705037415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1604.0,
|
|
"completions/max_terminated_length": 1604.0,
|
|
"completions/mean_length": 701.421875,
|
|
"completions/mean_terminated_length": 701.421875,
|
|
"completions/min_length": 316.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"entropy": 0.708191528916359,
|
|
"epoch": 0.22675736961451248,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.00698961055604741,
|
|
"learning_rate": 7.738095238095238e-07,
|
|
"loss": -0.0018,
|
|
"num_tokens": 23702196.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6040127277374268,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.109375,
|
|
"rewards/format_reward_func/std": 0.2083333432674408,
|
|
"step": 400,
|
|
"step_time": 55.5334142036736
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1712.0,
|
|
"completions/max_terminated_length": 1712.0,
|
|
"completions/mean_length": 774.65625,
|
|
"completions/mean_terminated_length": 774.65625,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.886969679966569,
|
|
"epoch": 0.22732426303854875,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.007826441855286248,
|
|
"learning_rate": 7.732426303854876e-07,
|
|
"loss": -0.0255,
|
|
"num_tokens": 23763430.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.528690755367279,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 401,
|
|
"step_time": 59.97146640624851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1648.0,
|
|
"completions/max_terminated_length": 1648.0,
|
|
"completions/mean_length": 803.59375,
|
|
"completions/mean_terminated_length": 803.59375,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.810698663815856,
|
|
"epoch": 0.22789115646258504,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.012139400845626369,
|
|
"learning_rate": 7.726757369614512e-07,
|
|
"loss": -0.0093,
|
|
"num_tokens": 23827900.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 402,
|
|
"step_time": 58.216640568338335
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1508.0,
|
|
"completions/max_terminated_length": 1508.0,
|
|
"completions/mean_length": 700.5,
|
|
"completions/mean_terminated_length": 700.5,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.7563704717904329,
|
|
"epoch": 0.22845804988662133,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.01042965950910002,
|
|
"learning_rate": 7.721088435374149e-07,
|
|
"loss": 0.0104,
|
|
"num_tokens": 23881908.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 403,
|
|
"step_time": 52.62817795109004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2535.0,
|
|
"completions/max_terminated_length": 2535.0,
|
|
"completions/mean_length": 730.875,
|
|
"completions/mean_terminated_length": 730.875,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.6416552513837814,
|
|
"epoch": 0.2290249433106576,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.012166628352133557,
|
|
"learning_rate": 7.715419501133786e-07,
|
|
"loss": 0.0248,
|
|
"num_tokens": 23940236.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7911965847015381,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 404,
|
|
"step_time": 94.3658363269642
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/max_terminated_length": 982.0,
|
|
"completions/mean_length": 618.953125,
|
|
"completions/mean_terminated_length": 618.953125,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 1.278845801949501,
|
|
"epoch": 0.22959183673469388,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.015799203421920538,
|
|
"learning_rate": 7.709750566893424e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 23992065.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 405,
|
|
"step_time": 34.95195194054395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1614.0,
|
|
"completions/max_terminated_length": 1614.0,
|
|
"completions/mean_length": 777.9375,
|
|
"completions/mean_terminated_length": 777.9375,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 0.7842423766851425,
|
|
"epoch": 0.23015873015873015,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.006712102811434306,
|
|
"learning_rate": 7.704081632653061e-07,
|
|
"loss": -0.0187,
|
|
"num_tokens": 24050733.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.6016985774040222,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 406,
|
|
"step_time": 57.17906874511391
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1314.0,
|
|
"completions/max_terminated_length": 1314.0,
|
|
"completions/mean_length": 809.640625,
|
|
"completions/mean_terminated_length": 809.640625,
|
|
"completions/min_length": 372.0,
|
|
"completions/min_terminated_length": 372.0,
|
|
"entropy": 0.7890525497496128,
|
|
"epoch": 0.23072562358276644,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.23046875,
|
|
"kl": 0.0075322547781979665,
|
|
"learning_rate": 7.698412698412699e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 24113734.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 407,
|
|
"step_time": 46.28891652356833
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1545.0,
|
|
"completions/max_terminated_length": 1545.0,
|
|
"completions/mean_length": 834.359375,
|
|
"completions/mean_terminated_length": 834.359375,
|
|
"completions/min_length": 473.0,
|
|
"completions/min_terminated_length": 473.0,
|
|
"entropy": 0.7317435592412949,
|
|
"epoch": 0.23129251700680273,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.018764967753668316,
|
|
"learning_rate": 7.692743764172335e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 24176309.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 408,
|
|
"step_time": 53.98831983841956
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1609.0,
|
|
"completions/max_terminated_length": 1609.0,
|
|
"completions/mean_length": 939.53125,
|
|
"completions/mean_terminated_length": 939.53125,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.6552373021841049,
|
|
"epoch": 0.231859410430839,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3203125,
|
|
"kl": 0.008924480978748761,
|
|
"learning_rate": 7.687074829931972e-07,
|
|
"loss": 0.0026,
|
|
"num_tokens": 24247775.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4787135720252991,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 409,
|
|
"step_time": 56.90790192503482
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1414.0,
|
|
"completions/max_terminated_length": 1414.0,
|
|
"completions/mean_length": 703.5625,
|
|
"completions/mean_terminated_length": 703.5625,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.769635409116745,
|
|
"epoch": 0.23242630385487528,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.007143637565604877,
|
|
"learning_rate": 7.68140589569161e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 24306219.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 410,
|
|
"step_time": 51.87743657454848
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1533.0,
|
|
"completions/max_terminated_length": 1533.0,
|
|
"completions/mean_length": 877.0,
|
|
"completions/mean_terminated_length": 877.0,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.6306070797145367,
|
|
"epoch": 0.23299319727891157,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.42578125,
|
|
"kl": 0.005516977093066089,
|
|
"learning_rate": 7.675736961451246e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 24373091.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 411,
|
|
"step_time": 54.269387434236705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1518.0,
|
|
"completions/max_terminated_length": 1518.0,
|
|
"completions/mean_length": 737.828125,
|
|
"completions/mean_terminated_length": 737.828125,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 0.8014046140015125,
|
|
"epoch": 0.23356009070294784,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"kl": 0.006993270544626284,
|
|
"learning_rate": 7.670068027210884e-07,
|
|
"loss": -0.008,
|
|
"num_tokens": 24429456.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7446901798248291,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 412,
|
|
"step_time": 53.018550055101514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1667.0,
|
|
"completions/max_terminated_length": 1667.0,
|
|
"completions/mean_length": 828.375,
|
|
"completions/mean_terminated_length": 828.375,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.7223452497273684,
|
|
"epoch": 0.23412698412698413,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.0052251661400077865,
|
|
"learning_rate": 7.664399092970521e-07,
|
|
"loss": -0.0082,
|
|
"num_tokens": 24491352.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.549169659614563,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.09375,
|
|
"rewards/format_reward_func/std": 0.19669894874095917,
|
|
"step": 413,
|
|
"step_time": 57.776623263955116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1483.0,
|
|
"completions/max_terminated_length": 1483.0,
|
|
"completions/mean_length": 830.328125,
|
|
"completions/mean_terminated_length": 830.328125,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.5297314096242189,
|
|
"epoch": 0.23469387755102042,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.011464409122709185,
|
|
"learning_rate": 7.658730158730159e-07,
|
|
"loss": 0.0144,
|
|
"num_tokens": 24554581.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6662480235099792,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 414,
|
|
"step_time": 52.02922047767788
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1472.0,
|
|
"completions/max_terminated_length": 1472.0,
|
|
"completions/mean_length": 751.3125,
|
|
"completions/mean_terminated_length": 751.3125,
|
|
"completions/min_length": 295.0,
|
|
"completions/min_terminated_length": 295.0,
|
|
"entropy": 0.9294396489858627,
|
|
"epoch": 0.23526077097505668,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.006961827792110853,
|
|
"learning_rate": 7.653061224489795e-07,
|
|
"loss": 0.0455,
|
|
"num_tokens": 24613457.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 415,
|
|
"step_time": 51.68498680740595
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1395.0,
|
|
"completions/max_terminated_length": 1395.0,
|
|
"completions/mean_length": 708.25,
|
|
"completions/mean_terminated_length": 708.25,
|
|
"completions/min_length": 271.0,
|
|
"completions/min_terminated_length": 271.0,
|
|
"entropy": 0.7721049040555954,
|
|
"epoch": 0.23582766439909297,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2177734375,
|
|
"kl": 0.006501360003312584,
|
|
"learning_rate": 7.647392290249433e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 24671417.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4495919644832611,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 416,
|
|
"step_time": 49.7948232954368
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1853.0,
|
|
"completions/max_terminated_length": 1853.0,
|
|
"completions/mean_length": 753.90625,
|
|
"completions/mean_terminated_length": 753.90625,
|
|
"completions/min_length": 203.0,
|
|
"completions/min_terminated_length": 203.0,
|
|
"entropy": 0.7060510329902172,
|
|
"epoch": 0.23639455782312926,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.296875,
|
|
"kl": 0.011613724156632088,
|
|
"learning_rate": 7.64172335600907e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 24731715.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 417,
|
|
"step_time": 65.67481262143701
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1218.0,
|
|
"completions/max_terminated_length": 1218.0,
|
|
"completions/mean_length": 648.90625,
|
|
"completions/mean_terminated_length": 648.90625,
|
|
"completions/min_length": 185.0,
|
|
"completions/min_terminated_length": 185.0,
|
|
"entropy": 0.9807036016136408,
|
|
"epoch": 0.23696145124716553,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.010876500498852693,
|
|
"learning_rate": 7.636054421768708e-07,
|
|
"loss": -0.0033,
|
|
"num_tokens": 24784693.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 418,
|
|
"step_time": 43.0141840679571
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1202.0,
|
|
"completions/max_terminated_length": 1202.0,
|
|
"completions/mean_length": 695.703125,
|
|
"completions/mean_terminated_length": 695.703125,
|
|
"completions/min_length": 268.0,
|
|
"completions/min_terminated_length": 268.0,
|
|
"entropy": 0.700308034196496,
|
|
"epoch": 0.23752834467120182,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.011756473148125224,
|
|
"learning_rate": 7.630385487528344e-07,
|
|
"loss": -0.0051,
|
|
"num_tokens": 24839458.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 419,
|
|
"step_time": 42.459158713929355
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1592.0,
|
|
"completions/max_terminated_length": 1592.0,
|
|
"completions/mean_length": 766.3125,
|
|
"completions/mean_terminated_length": 766.3125,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 0.8335917517542839,
|
|
"epoch": 0.23809523809523808,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3515625,
|
|
"kl": 0.007984273630427197,
|
|
"learning_rate": 7.624716553287982e-07,
|
|
"loss": -0.0152,
|
|
"num_tokens": 24899534.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5092765688896179,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 420,
|
|
"step_time": 56.33396484144032
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2090.0,
|
|
"completions/max_terminated_length": 2090.0,
|
|
"completions/mean_length": 784.921875,
|
|
"completions/mean_terminated_length": 784.921875,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.8108249437063932,
|
|
"epoch": 0.23866213151927437,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34765625,
|
|
"kl": 0.009463348469580524,
|
|
"learning_rate": 7.619047619047618e-07,
|
|
"loss": 0.0444,
|
|
"num_tokens": 24960313.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 421,
|
|
"step_time": 73.98894651699811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1189.0,
|
|
"completions/max_terminated_length": 1189.0,
|
|
"completions/mean_length": 680.71875,
|
|
"completions/mean_terminated_length": 680.71875,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.7651320341974497,
|
|
"epoch": 0.23922902494331066,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.009368772603920661,
|
|
"learning_rate": 7.613378684807256e-07,
|
|
"loss": -0.0073,
|
|
"num_tokens": 25013415.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7614454030990601,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 422,
|
|
"step_time": 42.23136337008327
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3049.0,
|
|
"completions/max_terminated_length": 3049.0,
|
|
"completions/mean_length": 744.796875,
|
|
"completions/mean_terminated_length": 744.796875,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.8581236302852631,
|
|
"epoch": 0.23979591836734693,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.60546875,
|
|
"kl": 0.016731746160076,
|
|
"learning_rate": 7.607709750566893e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 25071746.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 423,
|
|
"step_time": 116.03805009834468
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1223.0,
|
|
"completions/max_terminated_length": 1223.0,
|
|
"completions/mean_length": 708.03125,
|
|
"completions/mean_terminated_length": 708.03125,
|
|
"completions/min_length": 318.0,
|
|
"completions/min_terminated_length": 318.0,
|
|
"entropy": 0.8269235882908106,
|
|
"epoch": 0.24036281179138322,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.70703125,
|
|
"kl": 0.009408657337189652,
|
|
"learning_rate": 7.602040816326531e-07,
|
|
"loss": 0.0151,
|
|
"num_tokens": 25127276.0,
|
|
"reward": 0.421875,
|
|
"reward_std": 0.7777600884437561,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 424,
|
|
"step_time": 43.01042407378554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1607.0,
|
|
"completions/max_terminated_length": 1607.0,
|
|
"completions/mean_length": 730.46875,
|
|
"completions/mean_terminated_length": 730.46875,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.7529134526848793,
|
|
"epoch": 0.2409297052154195,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.00982889297301881,
|
|
"learning_rate": 7.596371882086168e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 25185394.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 425,
|
|
"step_time": 56.57126920670271
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1328.0,
|
|
"completions/max_terminated_length": 1328.0,
|
|
"completions/mean_length": 634.703125,
|
|
"completions/mean_terminated_length": 634.703125,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 1.2237685769796371,
|
|
"epoch": 0.24149659863945577,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.008560267873690464,
|
|
"learning_rate": 7.590702947845806e-07,
|
|
"loss": -0.0192,
|
|
"num_tokens": 25239415.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 426,
|
|
"step_time": 49.03863408975303
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1526.0,
|
|
"completions/max_terminated_length": 1526.0,
|
|
"completions/mean_length": 710.359375,
|
|
"completions/mean_terminated_length": 710.359375,
|
|
"completions/min_length": 244.0,
|
|
"completions/min_terminated_length": 244.0,
|
|
"entropy": 0.8540695123374462,
|
|
"epoch": 0.24206349206349206,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30078125,
|
|
"kl": 0.00960883658262901,
|
|
"learning_rate": 7.585034013605441e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 25296710.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6898858547210693,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 427,
|
|
"step_time": 54.59573978651315
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1398.0,
|
|
"completions/max_terminated_length": 1398.0,
|
|
"completions/mean_length": 796.84375,
|
|
"completions/mean_terminated_length": 796.84375,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.6341959312558174,
|
|
"epoch": 0.24263038548752835,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.42578125,
|
|
"kl": 0.00874024344375357,
|
|
"learning_rate": 7.579365079365078e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 25358604.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 428,
|
|
"step_time": 49.703380913473666
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1209.0,
|
|
"completions/max_terminated_length": 1209.0,
|
|
"completions/mean_length": 720.515625,
|
|
"completions/mean_terminated_length": 720.515625,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.8529839962720871,
|
|
"epoch": 0.24319727891156462,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.010199644399108365,
|
|
"learning_rate": 7.573696145124716e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 25414237.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 429,
|
|
"step_time": 42.44065989367664
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2672.0,
|
|
"completions/max_terminated_length": 2672.0,
|
|
"completions/mean_length": 770.734375,
|
|
"completions/mean_terminated_length": 770.734375,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 0.9049090035259724,
|
|
"epoch": 0.2437641723356009,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.006098392186686397,
|
|
"learning_rate": 7.568027210884353e-07,
|
|
"loss": 0.0221,
|
|
"num_tokens": 25476180.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 430,
|
|
"step_time": 99.39775457978249
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1474.0,
|
|
"completions/max_terminated_length": 1474.0,
|
|
"completions/mean_length": 709.59375,
|
|
"completions/mean_terminated_length": 709.59375,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.8665502294898033,
|
|
"epoch": 0.2443310657596372,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.017350808397168294,
|
|
"learning_rate": 7.562358276643991e-07,
|
|
"loss": -0.0523,
|
|
"num_tokens": 25531130.0,
|
|
"reward": 0.5546875,
|
|
"reward_std": 0.8550373315811157,
|
|
"rewards/accuracy_reward_func/mean": 0.5,
|
|
"rewards/accuracy_reward_func/std": 0.8728715777397156,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 431,
|
|
"step_time": 51.904258251190186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1440.0,
|
|
"completions/max_terminated_length": 1440.0,
|
|
"completions/mean_length": 713.40625,
|
|
"completions/mean_terminated_length": 713.40625,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.8070783540606499,
|
|
"epoch": 0.24489795918367346,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.7265625,
|
|
"kl": 0.010728749562986195,
|
|
"learning_rate": 7.556689342403628e-07,
|
|
"loss": -0.0235,
|
|
"num_tokens": 25586036.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 432,
|
|
"step_time": 50.71470108628273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2009.0,
|
|
"completions/max_terminated_length": 2009.0,
|
|
"completions/mean_length": 799.0,
|
|
"completions/mean_terminated_length": 799.0,
|
|
"completions/min_length": 239.0,
|
|
"completions/min_terminated_length": 239.0,
|
|
"entropy": 0.5964866243302822,
|
|
"epoch": 0.24546485260770975,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.0044653531949734315,
|
|
"learning_rate": 7.551020408163265e-07,
|
|
"loss": -0.0234,
|
|
"num_tokens": 25648468.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8199823498725891,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 433,
|
|
"step_time": 73.19168344326317
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1266.0,
|
|
"completions/max_terminated_length": 1266.0,
|
|
"completions/mean_length": 702.15625,
|
|
"completions/mean_terminated_length": 702.15625,
|
|
"completions/min_length": 286.0,
|
|
"completions/min_terminated_length": 286.0,
|
|
"entropy": 0.996532890945673,
|
|
"epoch": 0.24603174603174602,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.498046875,
|
|
"kl": 0.010547072102781385,
|
|
"learning_rate": 7.545351473922901e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 25712806.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6159563660621643,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 434,
|
|
"step_time": 52.20338438823819
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1473.0,
|
|
"completions/max_terminated_length": 1473.0,
|
|
"completions/mean_length": 796.34375,
|
|
"completions/mean_terminated_length": 796.34375,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.8812602311372757,
|
|
"epoch": 0.2465986394557823,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.0062492226279573515,
|
|
"learning_rate": 7.539682539682539e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 25773204.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.7759642601013184,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 435,
|
|
"step_time": 54.18101462442428
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1388.0,
|
|
"completions/mean_length": 809.109375,
|
|
"completions/mean_terminated_length": 756.9365234375,
|
|
"completions/min_length": 225.0,
|
|
"completions/min_terminated_length": 225.0,
|
|
"entropy": 0.8817069754004478,
|
|
"epoch": 0.2471655328798186,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.010766757986857556,
|
|
"learning_rate": 7.534013605442176e-07,
|
|
"loss": 0.0167,
|
|
"num_tokens": 25834115.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 436,
|
|
"step_time": 170.6649263864383
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1468.0,
|
|
"completions/max_terminated_length": 1468.0,
|
|
"completions/mean_length": 763.84375,
|
|
"completions/mean_terminated_length": 763.84375,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.7432105969637632,
|
|
"epoch": 0.24773242630385486,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.396484375,
|
|
"kl": 0.006353874458000064,
|
|
"learning_rate": 7.528344671201814e-07,
|
|
"loss": -0.0116,
|
|
"num_tokens": 25892729.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 437,
|
|
"step_time": 52.915428700856864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1375.0,
|
|
"completions/max_terminated_length": 1375.0,
|
|
"completions/mean_length": 864.5625,
|
|
"completions/mean_terminated_length": 864.5625,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.6446567680686712,
|
|
"epoch": 0.24829931972789115,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.005845273204613477,
|
|
"learning_rate": 7.522675736961451e-07,
|
|
"loss": 0.0277,
|
|
"num_tokens": 25957949.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.8614310622215271,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 438,
|
|
"step_time": 47.994716121815145
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 655.546875,
|
|
"completions/mean_terminated_length": 655.546875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.6887895595282316,
|
|
"epoch": 0.24886621315192745,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.013109318068018183,
|
|
"learning_rate": 7.517006802721088e-07,
|
|
"loss": 0.0194,
|
|
"num_tokens": 26011384.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 439,
|
|
"step_time": 47.03840684890747
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1645.0,
|
|
"completions/max_terminated_length": 1645.0,
|
|
"completions/mean_length": 670.484375,
|
|
"completions/mean_terminated_length": 670.484375,
|
|
"completions/min_length": 373.0,
|
|
"completions/min_terminated_length": 373.0,
|
|
"entropy": 0.866967786103487,
|
|
"epoch": 0.2494331065759637,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.008332332065037917,
|
|
"learning_rate": 7.511337868480725e-07,
|
|
"loss": -0.0031,
|
|
"num_tokens": 26064231.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 440,
|
|
"step_time": 57.855160099454224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2774.0,
|
|
"completions/max_terminated_length": 2774.0,
|
|
"completions/mean_length": 838.234375,
|
|
"completions/mean_terminated_length": 838.234375,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.8465369511395693,
|
|
"epoch": 0.25,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.357421875,
|
|
"kl": 0.010944681576802395,
|
|
"learning_rate": 7.505668934240363e-07,
|
|
"loss": -0.0077,
|
|
"num_tokens": 26128918.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 441,
|
|
"step_time": 103.54677613079548
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1499.0,
|
|
"completions/max_terminated_length": 1499.0,
|
|
"completions/mean_length": 682.53125,
|
|
"completions/mean_terminated_length": 682.53125,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.8602134734392166,
|
|
"epoch": 0.25056689342403626,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.009746387848281302,
|
|
"learning_rate": 7.5e-07,
|
|
"loss": 0.0083,
|
|
"num_tokens": 26181264.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 442,
|
|
"step_time": 52.056344939395785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1235.0,
|
|
"completions/max_terminated_length": 1235.0,
|
|
"completions/mean_length": 845.640625,
|
|
"completions/mean_terminated_length": 845.640625,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.9107060059905052,
|
|
"epoch": 0.2511337868480726,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.008831897255731747,
|
|
"learning_rate": 7.494331065759637e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 26249217.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4734388589859009,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 443,
|
|
"step_time": 45.33469169680029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1589.0,
|
|
"completions/max_terminated_length": 1589.0,
|
|
"completions/mean_length": 779.53125,
|
|
"completions/mean_terminated_length": 779.53125,
|
|
"completions/min_length": 241.0,
|
|
"completions/min_terminated_length": 241.0,
|
|
"entropy": 0.7985699214041233,
|
|
"epoch": 0.25170068027210885,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.009703594012535177,
|
|
"learning_rate": 7.488662131519274e-07,
|
|
"loss": -0.0058,
|
|
"num_tokens": 26308731.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 444,
|
|
"step_time": 55.171146512031555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2655.0,
|
|
"completions/max_terminated_length": 2655.0,
|
|
"completions/mean_length": 802.5625,
|
|
"completions/mean_terminated_length": 802.5625,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.7849292419850826,
|
|
"epoch": 0.2522675736961451,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.011236966223805211,
|
|
"learning_rate": 7.482993197278912e-07,
|
|
"loss": -0.0053,
|
|
"num_tokens": 26372239.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 445,
|
|
"step_time": 99.07032821141183
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1770.0,
|
|
"completions/max_terminated_length": 1770.0,
|
|
"completions/mean_length": 776.09375,
|
|
"completions/mean_terminated_length": 776.09375,
|
|
"completions/min_length": 382.0,
|
|
"completions/min_terminated_length": 382.0,
|
|
"entropy": 0.7460565585643053,
|
|
"epoch": 0.2528344671201814,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.007368704638793133,
|
|
"learning_rate": 7.477324263038548e-07,
|
|
"loss": 0.0465,
|
|
"num_tokens": 26431189.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 446,
|
|
"step_time": 62.11852004285902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/max_terminated_length": 1283.0,
|
|
"completions/mean_length": 637.84375,
|
|
"completions/mean_terminated_length": 637.84375,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 0.7619290817528963,
|
|
"epoch": 0.2534013605442177,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66796875,
|
|
"kl": 0.09098732945858501,
|
|
"learning_rate": 7.471655328798186e-07,
|
|
"loss": 0.0279,
|
|
"num_tokens": 26485547.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.7466442584991455,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 447,
|
|
"step_time": 46.95477893482894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1608.0,
|
|
"completions/max_terminated_length": 1608.0,
|
|
"completions/mean_length": 776.390625,
|
|
"completions/mean_terminated_length": 776.390625,
|
|
"completions/min_length": 263.0,
|
|
"completions/min_terminated_length": 263.0,
|
|
"entropy": 0.8864205647259951,
|
|
"epoch": 0.25396825396825395,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.007614424350322224,
|
|
"learning_rate": 7.465986394557823e-07,
|
|
"loss": 0.0319,
|
|
"num_tokens": 26545540.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 448,
|
|
"step_time": 56.151034818962216
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1619.0,
|
|
"completions/max_terminated_length": 1619.0,
|
|
"completions/mean_length": 751.96875,
|
|
"completions/mean_terminated_length": 751.96875,
|
|
"completions/min_length": 272.0,
|
|
"completions/min_terminated_length": 272.0,
|
|
"entropy": 0.8478775098919868,
|
|
"epoch": 0.2545351473922903,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.011353968890034594,
|
|
"learning_rate": 7.46031746031746e-07,
|
|
"loss": -0.029,
|
|
"num_tokens": 26604482.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6423677802085876,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 449,
|
|
"step_time": 56.435223148204386
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1599.0,
|
|
"completions/max_terminated_length": 1599.0,
|
|
"completions/mean_length": 873.046875,
|
|
"completions/mean_terminated_length": 873.046875,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.7115080244839191,
|
|
"epoch": 0.25510204081632654,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.296875,
|
|
"kl": 0.006389436239260249,
|
|
"learning_rate": 7.454648526077098e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 26671509.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5261042714118958,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 450,
|
|
"step_time": 56.23115598317236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1359.0,
|
|
"completions/max_terminated_length": 1359.0,
|
|
"completions/mean_length": 739.125,
|
|
"completions/mean_terminated_length": 739.125,
|
|
"completions/min_length": 271.0,
|
|
"completions/min_terminated_length": 271.0,
|
|
"entropy": 0.7353005595505238,
|
|
"epoch": 0.2556689342403628,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.01698808226501569,
|
|
"learning_rate": 7.448979591836734e-07,
|
|
"loss": -0.0075,
|
|
"num_tokens": 26728517.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 451,
|
|
"step_time": 47.689306191168725
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2467.0,
|
|
"completions/max_terminated_length": 2467.0,
|
|
"completions/mean_length": 838.859375,
|
|
"completions/mean_terminated_length": 838.859375,
|
|
"completions/min_length": 280.0,
|
|
"completions/min_terminated_length": 280.0,
|
|
"entropy": 0.7768061272799969,
|
|
"epoch": 0.2562358276643991,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.00658772730093915,
|
|
"learning_rate": 7.443310657596371e-07,
|
|
"loss": -0.0074,
|
|
"num_tokens": 26793916.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 452,
|
|
"step_time": 91.44622963294387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1435.0,
|
|
"completions/max_terminated_length": 1435.0,
|
|
"completions/mean_length": 741.171875,
|
|
"completions/mean_terminated_length": 741.171875,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.8883970566093922,
|
|
"epoch": 0.2568027210884354,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.009081590556888841,
|
|
"learning_rate": 7.437641723356008e-07,
|
|
"loss": 0.0133,
|
|
"num_tokens": 26851351.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 453,
|
|
"step_time": 50.2193358335644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1412.0,
|
|
"completions/max_terminated_length": 1412.0,
|
|
"completions/mean_length": 731.71875,
|
|
"completions/mean_terminated_length": 731.71875,
|
|
"completions/min_length": 351.0,
|
|
"completions/min_terminated_length": 351.0,
|
|
"entropy": 0.8555888086557388,
|
|
"epoch": 0.25736961451247165,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.007854102237615734,
|
|
"learning_rate": 7.431972789115646e-07,
|
|
"loss": -0.0209,
|
|
"num_tokens": 26907917.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 454,
|
|
"step_time": 49.07495683711022
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1663.0,
|
|
"completions/max_terminated_length": 1663.0,
|
|
"completions/mean_length": 789.453125,
|
|
"completions/mean_terminated_length": 789.453125,
|
|
"completions/min_length": 322.0,
|
|
"completions/min_terminated_length": 322.0,
|
|
"entropy": 0.8106684256345034,
|
|
"epoch": 0.25793650793650796,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.006390364105754998,
|
|
"learning_rate": 7.426303854875283e-07,
|
|
"loss": 0.0256,
|
|
"num_tokens": 26969818.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 455,
|
|
"step_time": 58.62766888923943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1595.0,
|
|
"completions/max_terminated_length": 1595.0,
|
|
"completions/mean_length": 811.734375,
|
|
"completions/mean_terminated_length": 811.734375,
|
|
"completions/min_length": 389.0,
|
|
"completions/min_terminated_length": 389.0,
|
|
"entropy": 0.8144347574561834,
|
|
"epoch": 0.2585034013605442,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.006740940531017259,
|
|
"learning_rate": 7.420634920634921e-07,
|
|
"loss": -0.0019,
|
|
"num_tokens": 27030761.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 456,
|
|
"step_time": 55.83527867961675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1587.0,
|
|
"completions/max_terminated_length": 1587.0,
|
|
"completions/mean_length": 800.796875,
|
|
"completions/mean_terminated_length": 800.796875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.7810667101293802,
|
|
"epoch": 0.2590702947845805,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.013272675278130919,
|
|
"learning_rate": 7.414965986394558e-07,
|
|
"loss": 0.0197,
|
|
"num_tokens": 27093036.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3398725986480713,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 457,
|
|
"step_time": 55.82849807012826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1872.0,
|
|
"completions/max_terminated_length": 1872.0,
|
|
"completions/mean_length": 834.578125,
|
|
"completions/mean_terminated_length": 834.578125,
|
|
"completions/min_length": 476.0,
|
|
"completions/min_terminated_length": 476.0,
|
|
"entropy": 0.6509952079504728,
|
|
"epoch": 0.25963718820861675,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"kl": 0.007395024884317536,
|
|
"learning_rate": 7.409297052154194e-07,
|
|
"loss": 0.0147,
|
|
"num_tokens": 27158881.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.665922224521637,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 458,
|
|
"step_time": 67.85225706454366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1424.0,
|
|
"completions/max_terminated_length": 1424.0,
|
|
"completions/mean_length": 750.28125,
|
|
"completions/mean_terminated_length": 750.28125,
|
|
"completions/min_length": 294.0,
|
|
"completions/min_terminated_length": 294.0,
|
|
"entropy": 0.7547367867082357,
|
|
"epoch": 0.2602040816326531,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.01647935227083508,
|
|
"learning_rate": 7.403628117913831e-07,
|
|
"loss": -0.0065,
|
|
"num_tokens": 27217395.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 459,
|
|
"step_time": 51.251739129424095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1394.0,
|
|
"completions/max_terminated_length": 1394.0,
|
|
"completions/mean_length": 619.078125,
|
|
"completions/mean_terminated_length": 619.078125,
|
|
"completions/min_length": 238.0,
|
|
"completions/min_terminated_length": 238.0,
|
|
"entropy": 0.6322919260710478,
|
|
"epoch": 0.26077097505668934,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.006262985305511393,
|
|
"learning_rate": 7.397959183673469e-07,
|
|
"loss": -0.0019,
|
|
"num_tokens": 27269880.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 460,
|
|
"step_time": 49.96796751022339
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1637.0,
|
|
"completions/max_terminated_length": 1637.0,
|
|
"completions/mean_length": 754.65625,
|
|
"completions/mean_terminated_length": 754.65625,
|
|
"completions/min_length": 138.0,
|
|
"completions/min_terminated_length": 138.0,
|
|
"entropy": 0.7624211218208075,
|
|
"epoch": 0.2613378684807256,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.006042953609721735,
|
|
"learning_rate": 7.392290249433106e-07,
|
|
"loss": -0.0119,
|
|
"num_tokens": 27327338.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 461,
|
|
"step_time": 56.916788490489125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1305.0,
|
|
"completions/max_terminated_length": 1305.0,
|
|
"completions/mean_length": 748.109375,
|
|
"completions/mean_terminated_length": 748.109375,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.768423568457365,
|
|
"epoch": 0.2619047619047619,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.020247375447070226,
|
|
"learning_rate": 7.386621315192744e-07,
|
|
"loss": 0.0153,
|
|
"num_tokens": 27385721.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 462,
|
|
"step_time": 45.881373840384185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1515.0,
|
|
"completions/max_terminated_length": 1515.0,
|
|
"completions/mean_length": 803.25,
|
|
"completions/mean_terminated_length": 803.25,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.7130127977579832,
|
|
"epoch": 0.2624716553287982,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.010153452487429604,
|
|
"learning_rate": 7.380952380952381e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 27447729.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7295067310333252,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 463,
|
|
"step_time": 53.21283014770597
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1686.0,
|
|
"completions/max_terminated_length": 1686.0,
|
|
"completions/mean_length": 718.5,
|
|
"completions/mean_terminated_length": 718.5,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.858843918889761,
|
|
"epoch": 0.26303854875283444,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.416015625,
|
|
"kl": 0.008995608412078582,
|
|
"learning_rate": 7.375283446712018e-07,
|
|
"loss": 0.0028,
|
|
"num_tokens": 27503361.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 464,
|
|
"step_time": 59.50413880310953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1458.0,
|
|
"completions/max_terminated_length": 1458.0,
|
|
"completions/mean_length": 780.703125,
|
|
"completions/mean_terminated_length": 780.703125,
|
|
"completions/min_length": 461.0,
|
|
"completions/min_terminated_length": 461.0,
|
|
"entropy": 0.8416282907128334,
|
|
"epoch": 0.26360544217687076,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.026874932344071567,
|
|
"learning_rate": 7.369614512471655e-07,
|
|
"loss": 0.0155,
|
|
"num_tokens": 27564470.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 465,
|
|
"step_time": 51.26821294892579
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1280.0,
|
|
"completions/max_terminated_length": 1280.0,
|
|
"completions/mean_length": 720.125,
|
|
"completions/mean_terminated_length": 720.125,
|
|
"completions/min_length": 318.0,
|
|
"completions/min_terminated_length": 318.0,
|
|
"entropy": 0.9076618254184723,
|
|
"epoch": 0.264172335600907,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.009675825334852561,
|
|
"learning_rate": 7.363945578231292e-07,
|
|
"loss": -0.0357,
|
|
"num_tokens": 27621134.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 466,
|
|
"step_time": 45.1732674036175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1632.0,
|
|
"completions/max_terminated_length": 1632.0,
|
|
"completions/mean_length": 690.375,
|
|
"completions/mean_terminated_length": 690.375,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.8111182861030102,
|
|
"epoch": 0.2647392290249433,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.640625,
|
|
"kl": 0.007601130113471299,
|
|
"learning_rate": 7.358276643990929e-07,
|
|
"loss": -0.0126,
|
|
"num_tokens": 27677982.0,
|
|
"reward": 0.421875,
|
|
"reward_std": 0.8555085062980652,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 467,
|
|
"step_time": 58.271796913817525
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1070.0,
|
|
"completions/max_terminated_length": 1070.0,
|
|
"completions/mean_length": 709.578125,
|
|
"completions/mean_terminated_length": 709.578125,
|
|
"completions/min_length": 248.0,
|
|
"completions/min_terminated_length": 248.0,
|
|
"entropy": 0.9625997617840767,
|
|
"epoch": 0.2653061224489796,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.008367281930986792,
|
|
"learning_rate": 7.352607709750567e-07,
|
|
"loss": -0.0035,
|
|
"num_tokens": 27736035.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 468,
|
|
"step_time": 38.30163570214063
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1597.0,
|
|
"completions/max_terminated_length": 1597.0,
|
|
"completions/mean_length": 630.125,
|
|
"completions/mean_terminated_length": 630.125,
|
|
"completions/min_length": 39.0,
|
|
"completions/min_terminated_length": 39.0,
|
|
"entropy": 1.0288138017058372,
|
|
"epoch": 0.26587301587301587,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6171875,
|
|
"kl": 0.009008562250528485,
|
|
"learning_rate": 7.346938775510204e-07,
|
|
"loss": -0.0109,
|
|
"num_tokens": 27786283.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 469,
|
|
"step_time": 56.333386664278805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1656.0,
|
|
"completions/max_terminated_length": 1656.0,
|
|
"completions/mean_length": 882.765625,
|
|
"completions/mean_terminated_length": 882.765625,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.6477733794599771,
|
|
"epoch": 0.26643990929705214,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.400390625,
|
|
"kl": 0.004466563317691907,
|
|
"learning_rate": 7.34126984126984e-07,
|
|
"loss": 0.0068,
|
|
"num_tokens": 27854572.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 470,
|
|
"step_time": 58.486423769034445
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 788.140625,
|
|
"completions/mean_terminated_length": 788.140625,
|
|
"completions/min_length": 306.0,
|
|
"completions/min_terminated_length": 306.0,
|
|
"entropy": 0.6794374939054251,
|
|
"epoch": 0.26700680272108845,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.013015237913350575,
|
|
"learning_rate": 7.335600907029478e-07,
|
|
"loss": 0.0045,
|
|
"num_tokens": 27914869.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6422230005264282,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 471,
|
|
"step_time": 51.77089131716639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/max_terminated_length": 1283.0,
|
|
"completions/mean_length": 656.015625,
|
|
"completions/mean_terminated_length": 656.015625,
|
|
"completions/min_length": 200.0,
|
|
"completions/min_terminated_length": 200.0,
|
|
"entropy": 0.8764982726424932,
|
|
"epoch": 0.2675736961451247,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.00789091837941669,
|
|
"learning_rate": 7.329931972789115e-07,
|
|
"loss": 0.0068,
|
|
"num_tokens": 27968078.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 472,
|
|
"step_time": 46.43677334673703
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/max_terminated_length": 1283.0,
|
|
"completions/mean_length": 636.46875,
|
|
"completions/mean_terminated_length": 636.46875,
|
|
"completions/min_length": 293.0,
|
|
"completions/min_terminated_length": 293.0,
|
|
"entropy": 0.9508518874645233,
|
|
"epoch": 0.268140589569161,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.007630579173564911,
|
|
"learning_rate": 7.324263038548753e-07,
|
|
"loss": 0.0161,
|
|
"num_tokens": 28019332.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6412084698677063,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 473,
|
|
"step_time": 45.04764767270535
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1573.0,
|
|
"completions/max_terminated_length": 1573.0,
|
|
"completions/mean_length": 810.09375,
|
|
"completions/mean_terminated_length": 810.09375,
|
|
"completions/min_length": 403.0,
|
|
"completions/min_terminated_length": 403.0,
|
|
"entropy": 0.7245715744793415,
|
|
"epoch": 0.2687074829931973,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.35546875,
|
|
"kl": 0.007624058547662571,
|
|
"learning_rate": 7.318594104308389e-07,
|
|
"loss": 0.0081,
|
|
"num_tokens": 28082250.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 474,
|
|
"step_time": 54.913598663173616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2120.0,
|
|
"completions/max_terminated_length": 2120.0,
|
|
"completions/mean_length": 789.015625,
|
|
"completions/mean_terminated_length": 789.015625,
|
|
"completions/min_length": 411.0,
|
|
"completions/min_terminated_length": 411.0,
|
|
"entropy": 0.8914128709584475,
|
|
"epoch": 0.26927437641723356,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.43359375,
|
|
"kl": 0.005593473179033026,
|
|
"learning_rate": 7.312925170068027e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 28143955.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 475,
|
|
"step_time": 76.40175187215209
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1940.0,
|
|
"completions/max_terminated_length": 1940.0,
|
|
"completions/mean_length": 793.96875,
|
|
"completions/mean_terminated_length": 793.96875,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.796158604323864,
|
|
"epoch": 0.2698412698412698,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.38671875,
|
|
"kl": 0.00707636731385719,
|
|
"learning_rate": 7.307256235827664e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 28207857.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 476,
|
|
"step_time": 70.26205126568675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1342.0,
|
|
"completions/max_terminated_length": 1342.0,
|
|
"completions/mean_length": 654.921875,
|
|
"completions/mean_terminated_length": 654.921875,
|
|
"completions/min_length": 54.0,
|
|
"completions/min_terminated_length": 54.0,
|
|
"entropy": 0.7878446727991104,
|
|
"epoch": 0.27040816326530615,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.02065294745261781,
|
|
"learning_rate": 7.301587301587301e-07,
|
|
"loss": -0.0129,
|
|
"num_tokens": 28260068.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 477,
|
|
"step_time": 47.16918959002942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1331.0,
|
|
"completions/max_terminated_length": 1331.0,
|
|
"completions/mean_length": 760.125,
|
|
"completions/mean_terminated_length": 760.125,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.6362970042973757,
|
|
"epoch": 0.2709750566893424,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.009445872463402338,
|
|
"learning_rate": 7.295918367346938e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 28321076.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6884463429450989,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 478,
|
|
"step_time": 48.2730901427567
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1842.0,
|
|
"completions/max_terminated_length": 1842.0,
|
|
"completions/mean_length": 881.375,
|
|
"completions/mean_terminated_length": 881.375,
|
|
"completions/min_length": 434.0,
|
|
"completions/min_terminated_length": 434.0,
|
|
"entropy": 0.7171924952417612,
|
|
"epoch": 0.27154195011337867,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.416015625,
|
|
"kl": 0.007329938503971789,
|
|
"learning_rate": 7.290249433106576e-07,
|
|
"loss": 0.0255,
|
|
"num_tokens": 28389620.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 479,
|
|
"step_time": 64.85122813191265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1747.0,
|
|
"completions/max_terminated_length": 1747.0,
|
|
"completions/mean_length": 767.296875,
|
|
"completions/mean_terminated_length": 767.296875,
|
|
"completions/min_length": 331.0,
|
|
"completions/min_terminated_length": 331.0,
|
|
"entropy": 0.6492326371371746,
|
|
"epoch": 0.272108843537415,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.01491025963332504,
|
|
"learning_rate": 7.284580498866213e-07,
|
|
"loss": -0.0128,
|
|
"num_tokens": 28450863.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6884463429450989,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 480,
|
|
"step_time": 61.70564486365765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1431.0,
|
|
"completions/max_terminated_length": 1431.0,
|
|
"completions/mean_length": 779.890625,
|
|
"completions/mean_terminated_length": 779.890625,
|
|
"completions/min_length": 58.0,
|
|
"completions/min_terminated_length": 58.0,
|
|
"entropy": 0.8773079253733158,
|
|
"epoch": 0.27267573696145125,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.011495613274746574,
|
|
"learning_rate": 7.278911564625851e-07,
|
|
"loss": 0.0196,
|
|
"num_tokens": 28510808.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 481,
|
|
"step_time": 49.98250643257052
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1550.0,
|
|
"completions/max_terminated_length": 1550.0,
|
|
"completions/mean_length": 706.140625,
|
|
"completions/mean_terminated_length": 706.140625,
|
|
"completions/min_length": 222.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"entropy": 1.0600604228675365,
|
|
"epoch": 0.2732426303854875,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.006201082011102699,
|
|
"learning_rate": 7.273242630385487e-07,
|
|
"loss": -0.0074,
|
|
"num_tokens": 28567169.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 482,
|
|
"step_time": 55.084377290681005
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1512.0,
|
|
"completions/max_terminated_length": 1512.0,
|
|
"completions/mean_length": 752.453125,
|
|
"completions/mean_terminated_length": 752.453125,
|
|
"completions/min_length": 304.0,
|
|
"completions/min_terminated_length": 304.0,
|
|
"entropy": 0.7403154652565718,
|
|
"epoch": 0.27380952380952384,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.033526878178236075,
|
|
"learning_rate": 7.267573696145124e-07,
|
|
"loss": -0.0007,
|
|
"num_tokens": 28624782.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 483,
|
|
"step_time": 52.685782368294895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1272.0,
|
|
"completions/max_terminated_length": 1272.0,
|
|
"completions/mean_length": 726.9375,
|
|
"completions/mean_terminated_length": 726.9375,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 1.061217077076435,
|
|
"epoch": 0.2743764172335601,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.00911310211813543,
|
|
"learning_rate": 7.261904761904761e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 28681074.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 484,
|
|
"step_time": 44.70914957206696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1217.0,
|
|
"completions/max_terminated_length": 1217.0,
|
|
"completions/mean_length": 756.390625,
|
|
"completions/mean_terminated_length": 756.390625,
|
|
"completions/min_length": 479.0,
|
|
"completions/min_terminated_length": 479.0,
|
|
"entropy": 0.7275263834744692,
|
|
"epoch": 0.27494331065759636,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.010503193392651156,
|
|
"learning_rate": 7.256235827664399e-07,
|
|
"loss": 0.0212,
|
|
"num_tokens": 28740579.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 485,
|
|
"step_time": 42.743415992707014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 655.421875,
|
|
"completions/mean_terminated_length": 655.421875,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 1.026012111455202,
|
|
"epoch": 0.2755102040816326,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.00818357955722604,
|
|
"learning_rate": 7.250566893424036e-07,
|
|
"loss": 0.0112,
|
|
"num_tokens": 28792062.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 486,
|
|
"step_time": 44.48652746528387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1281.0,
|
|
"completions/max_terminated_length": 1281.0,
|
|
"completions/mean_length": 701.90625,
|
|
"completions/mean_terminated_length": 701.90625,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 1.0500460527837276,
|
|
"epoch": 0.27607709750566894,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.310546875,
|
|
"kl": 0.011005967840901576,
|
|
"learning_rate": 7.244897959183674e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 28846352.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 487,
|
|
"step_time": 44.67506785038859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1616.0,
|
|
"completions/max_terminated_length": 1616.0,
|
|
"completions/mean_length": 719.8125,
|
|
"completions/mean_terminated_length": 719.8125,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 1.1547892428934574,
|
|
"epoch": 0.2766439909297052,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.007268138855579309,
|
|
"learning_rate": 7.239229024943311e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 28902276.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 488,
|
|
"step_time": 56.27806665096432
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1477.0,
|
|
"completions/max_terminated_length": 1477.0,
|
|
"completions/mean_length": 661.328125,
|
|
"completions/mean_terminated_length": 661.328125,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.7691992409527302,
|
|
"epoch": 0.27721088435374147,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58984375,
|
|
"kl": 0.008071009156992659,
|
|
"learning_rate": 7.233560090702947e-07,
|
|
"loss": -0.0153,
|
|
"num_tokens": 28953929.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6542273163795471,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 489,
|
|
"step_time": 52.05397119093686
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1409.0,
|
|
"completions/max_terminated_length": 1409.0,
|
|
"completions/mean_length": 721.1875,
|
|
"completions/mean_terminated_length": 721.1875,
|
|
"completions/min_length": 164.0,
|
|
"completions/min_terminated_length": 164.0,
|
|
"entropy": 0.8899144306778908,
|
|
"epoch": 0.2777777777777778,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.007417619170155376,
|
|
"learning_rate": 7.227891156462584e-07,
|
|
"loss": 0.0241,
|
|
"num_tokens": 29009837.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 490,
|
|
"step_time": 49.597440999001265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1460.0,
|
|
"completions/max_terminated_length": 1460.0,
|
|
"completions/mean_length": 715.09375,
|
|
"completions/mean_terminated_length": 715.09375,
|
|
"completions/min_length": 221.0,
|
|
"completions/min_terminated_length": 221.0,
|
|
"entropy": 0.7251012995839119,
|
|
"epoch": 0.27834467120181405,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.008479311072733253,
|
|
"learning_rate": 7.222222222222221e-07,
|
|
"loss": -0.0339,
|
|
"num_tokens": 29065331.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7099072337150574,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 491,
|
|
"step_time": 50.99935754854232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1225.0,
|
|
"completions/max_terminated_length": 1225.0,
|
|
"completions/mean_length": 792.296875,
|
|
"completions/mean_terminated_length": 792.296875,
|
|
"completions/min_length": 473.0,
|
|
"completions/min_terminated_length": 473.0,
|
|
"entropy": 0.8285106271505356,
|
|
"epoch": 0.2789115646258503,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.007081485047820024,
|
|
"learning_rate": 7.216553287981859e-07,
|
|
"loss": 0.0196,
|
|
"num_tokens": 29126846.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 492,
|
|
"step_time": 43.31351323239505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1736.0,
|
|
"completions/max_terminated_length": 1736.0,
|
|
"completions/mean_length": 812.609375,
|
|
"completions/mean_terminated_length": 812.609375,
|
|
"completions/min_length": 430.0,
|
|
"completions/min_terminated_length": 430.0,
|
|
"entropy": 0.7524502705782652,
|
|
"epoch": 0.27947845804988664,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.008174628295819275,
|
|
"learning_rate": 7.210884353741496e-07,
|
|
"loss": 0.0053,
|
|
"num_tokens": 29188581.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 493,
|
|
"step_time": 60.781097644008696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1398.0,
|
|
"completions/max_terminated_length": 1398.0,
|
|
"completions/mean_length": 770.734375,
|
|
"completions/mean_terminated_length": 770.734375,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.8369970675557852,
|
|
"epoch": 0.2800453514739229,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.010711344511946663,
|
|
"learning_rate": 7.205215419501134e-07,
|
|
"loss": 0.0061,
|
|
"num_tokens": 29248356.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 494,
|
|
"step_time": 48.85982634592801
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1397.0,
|
|
"completions/max_terminated_length": 1397.0,
|
|
"completions/mean_length": 726.390625,
|
|
"completions/mean_terminated_length": 726.390625,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 1.0261105597019196,
|
|
"epoch": 0.28061224489795916,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.0050688901683315635,
|
|
"learning_rate": 7.19954648526077e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 29307085.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6784669756889343,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 495,
|
|
"step_time": 49.4272082047537
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1442.0,
|
|
"completions/mean_length": 784.78125,
|
|
"completions/mean_terminated_length": 732.2222900390625,
|
|
"completions/min_length": 295.0,
|
|
"completions/min_terminated_length": 295.0,
|
|
"entropy": 0.996563445776701,
|
|
"epoch": 0.2811791383219955,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.005772793374489993,
|
|
"learning_rate": 7.193877551020408e-07,
|
|
"loss": -0.0064,
|
|
"num_tokens": 29368719.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 496,
|
|
"step_time": 172.76854544319212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1316.0,
|
|
"completions/max_terminated_length": 1316.0,
|
|
"completions/mean_length": 741.5625,
|
|
"completions/mean_terminated_length": 741.5625,
|
|
"completions/min_length": 198.0,
|
|
"completions/min_terminated_length": 198.0,
|
|
"entropy": 0.7578916009515524,
|
|
"epoch": 0.28174603174603174,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.65234375,
|
|
"kl": 0.008651686686789617,
|
|
"learning_rate": 7.188208616780045e-07,
|
|
"loss": 0.0025,
|
|
"num_tokens": 29428747.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 497,
|
|
"step_time": 50.83865497633815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1277.0,
|
|
"completions/max_terminated_length": 1277.0,
|
|
"completions/mean_length": 752.46875,
|
|
"completions/mean_terminated_length": 752.46875,
|
|
"completions/min_length": 461.0,
|
|
"completions/min_terminated_length": 461.0,
|
|
"entropy": 0.9528252314776182,
|
|
"epoch": 0.282312925170068,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.009568301378749311,
|
|
"learning_rate": 7.182539682539682e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 29488929.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 498,
|
|
"step_time": 44.66134562995285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1780.0,
|
|
"completions/max_terminated_length": 1780.0,
|
|
"completions/mean_length": 786.109375,
|
|
"completions/mean_terminated_length": 786.109375,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.7473367918282747,
|
|
"epoch": 0.2828798185941043,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.014012510364409536,
|
|
"learning_rate": 7.176870748299319e-07,
|
|
"loss": 0.0156,
|
|
"num_tokens": 29549768.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 499,
|
|
"step_time": 62.703040649183095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1286.0,
|
|
"completions/max_terminated_length": 1286.0,
|
|
"completions/mean_length": 755.1875,
|
|
"completions/mean_terminated_length": 755.1875,
|
|
"completions/min_length": 366.0,
|
|
"completions/min_terminated_length": 366.0,
|
|
"entropy": 0.9694885313510895,
|
|
"epoch": 0.2834467120181406,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.33984375,
|
|
"kl": 0.014857098081847653,
|
|
"learning_rate": 7.171201814058957e-07,
|
|
"loss": -0.0136,
|
|
"num_tokens": 29609892.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 500,
|
|
"step_time": 46.38259715773165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1193.0,
|
|
"completions/max_terminated_length": 1193.0,
|
|
"completions/mean_length": 621.21875,
|
|
"completions/mean_terminated_length": 621.21875,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.8720263168215752,
|
|
"epoch": 0.28401360544217685,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.73828125,
|
|
"kl": 0.014097395222052,
|
|
"learning_rate": 7.165532879818594e-07,
|
|
"loss": 0.0161,
|
|
"num_tokens": 29659370.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 501,
|
|
"step_time": 41.85722758155316
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1497.0,
|
|
"completions/max_terminated_length": 1497.0,
|
|
"completions/mean_length": 785.296875,
|
|
"completions/mean_terminated_length": 785.296875,
|
|
"completions/min_length": 403.0,
|
|
"completions/min_terminated_length": 403.0,
|
|
"entropy": 0.8356814160943031,
|
|
"epoch": 0.28458049886621317,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.005160079905181192,
|
|
"learning_rate": 7.159863945578231e-07,
|
|
"loss": 0.0253,
|
|
"num_tokens": 29720837.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6159563660621643,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 502,
|
|
"step_time": 53.16263134498149
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1193.0,
|
|
"completions/max_terminated_length": 1193.0,
|
|
"completions/mean_length": 754.671875,
|
|
"completions/mean_terminated_length": 754.671875,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.8363759834319353,
|
|
"epoch": 0.28514739229024944,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.005708936325390823,
|
|
"learning_rate": 7.154195011337868e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 29780648.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 503,
|
|
"step_time": 41.80383961182088
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1516.0,
|
|
"completions/max_terminated_length": 1516.0,
|
|
"completions/mean_length": 855.375,
|
|
"completions/mean_terminated_length": 855.375,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.6615287847816944,
|
|
"epoch": 0.2857142857142857,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.0060810370778199285,
|
|
"learning_rate": 7.148526077097506e-07,
|
|
"loss": -0.0129,
|
|
"num_tokens": 29844864.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5579067468643188,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 504,
|
|
"step_time": 53.0472785839811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1182.0,
|
|
"completions/mean_length": 762.5625,
|
|
"completions/mean_terminated_length": 709.6508178710938,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.7538333609700203,
|
|
"epoch": 0.286281179138322,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.00518970109988004,
|
|
"learning_rate": 7.142857142857143e-07,
|
|
"loss": -0.0075,
|
|
"num_tokens": 29901524.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 505,
|
|
"step_time": 169.49539405293763
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3136.0,
|
|
"completions/max_terminated_length": 3136.0,
|
|
"completions/mean_length": 860.40625,
|
|
"completions/mean_terminated_length": 860.40625,
|
|
"completions/min_length": 393.0,
|
|
"completions/min_terminated_length": 393.0,
|
|
"entropy": 0.6654835715889931,
|
|
"epoch": 0.2868480725623583,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.0035152445125277154,
|
|
"learning_rate": 7.13718820861678e-07,
|
|
"loss": -0.0308,
|
|
"num_tokens": 29966062.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 506,
|
|
"step_time": 119.62166095059365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1173.0,
|
|
"completions/max_terminated_length": 1173.0,
|
|
"completions/mean_length": 636.390625,
|
|
"completions/mean_terminated_length": 636.390625,
|
|
"completions/min_length": 275.0,
|
|
"completions/min_terminated_length": 275.0,
|
|
"entropy": 1.0220135562121868,
|
|
"epoch": 0.28741496598639454,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.012051342375343665,
|
|
"learning_rate": 7.131519274376417e-07,
|
|
"loss": -0.012,
|
|
"num_tokens": 30019751.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.47709178924560547,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 507,
|
|
"step_time": 42.66235166601837
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1924.0,
|
|
"completions/max_terminated_length": 1924.0,
|
|
"completions/mean_length": 675.296875,
|
|
"completions/mean_terminated_length": 675.296875,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 0.6998019386082888,
|
|
"epoch": 0.28798185941043086,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.275390625,
|
|
"kl": 0.01043865256360732,
|
|
"learning_rate": 7.125850340136053e-07,
|
|
"loss": -0.0048,
|
|
"num_tokens": 30072146.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 508,
|
|
"step_time": 67.1354058617726
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1435.0,
|
|
"completions/max_terminated_length": 1435.0,
|
|
"completions/mean_length": 816.109375,
|
|
"completions/mean_terminated_length": 816.109375,
|
|
"completions/min_length": 465.0,
|
|
"completions/min_terminated_length": 465.0,
|
|
"entropy": 0.8015304617583752,
|
|
"epoch": 0.2885487528344671,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.011067343672038987,
|
|
"learning_rate": 7.120181405895691e-07,
|
|
"loss": -0.0083,
|
|
"num_tokens": 30136289.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.7544510364532471,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 509,
|
|
"step_time": 50.518286121077836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1248.0,
|
|
"completions/max_terminated_length": 1248.0,
|
|
"completions/mean_length": 746.1875,
|
|
"completions/mean_terminated_length": 746.1875,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.9967562705278397,
|
|
"epoch": 0.2891156462585034,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.005694640116416849,
|
|
"learning_rate": 7.114512471655328e-07,
|
|
"loss": -0.0027,
|
|
"num_tokens": 30192069.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 510,
|
|
"step_time": 43.2562718084082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1545.0,
|
|
"completions/max_terminated_length": 1545.0,
|
|
"completions/mean_length": 762.40625,
|
|
"completions/mean_terminated_length": 762.40625,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.6715107001364231,
|
|
"epoch": 0.2896825396825397,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2392578125,
|
|
"kl": 0.004302979759813752,
|
|
"learning_rate": 7.108843537414966e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 30252527.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 511,
|
|
"step_time": 54.68684424832463
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1262.0,
|
|
"completions/max_terminated_length": 1262.0,
|
|
"completions/mean_length": 712.71875,
|
|
"completions/mean_terminated_length": 712.71875,
|
|
"completions/min_length": 257.0,
|
|
"completions/min_terminated_length": 257.0,
|
|
"entropy": 0.8312527053058147,
|
|
"epoch": 0.29024943310657597,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.007387642486719415,
|
|
"learning_rate": 7.103174603174603e-07,
|
|
"loss": 0.0113,
|
|
"num_tokens": 30309253.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6993034482002258,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 512,
|
|
"step_time": 44.56001595873386
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1211.0,
|
|
"completions/max_terminated_length": 1211.0,
|
|
"completions/mean_length": 765.9375,
|
|
"completions/mean_terminated_length": 765.9375,
|
|
"completions/min_length": 393.0,
|
|
"completions/min_terminated_length": 393.0,
|
|
"entropy": 0.6705945897847414,
|
|
"epoch": 0.29081632653061223,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.006404365805792622,
|
|
"learning_rate": 7.097505668934241e-07,
|
|
"loss": 0.0054,
|
|
"num_tokens": 30368809.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 513,
|
|
"step_time": 42.842675207182765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1818.0,
|
|
"completions/max_terminated_length": 1818.0,
|
|
"completions/mean_length": 744.125,
|
|
"completions/mean_terminated_length": 744.125,
|
|
"completions/min_length": 199.0,
|
|
"completions/min_terminated_length": 199.0,
|
|
"entropy": 0.9200004376471043,
|
|
"epoch": 0.29138321995464855,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"kl": 0.008822381729260087,
|
|
"learning_rate": 7.091836734693876e-07,
|
|
"loss": 0.0018,
|
|
"num_tokens": 30427825.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 514,
|
|
"step_time": 65.67067721299827
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1690.0,
|
|
"completions/max_terminated_length": 1690.0,
|
|
"completions/mean_length": 715.84375,
|
|
"completions/mean_terminated_length": 715.84375,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.9182790108025074,
|
|
"epoch": 0.2919501133786848,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.011332457012031227,
|
|
"learning_rate": 7.086167800453514e-07,
|
|
"loss": 0.024,
|
|
"num_tokens": 30486975.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 515,
|
|
"step_time": 61.574917116202414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1506.0,
|
|
"completions/max_terminated_length": 1506.0,
|
|
"completions/mean_length": 872.59375,
|
|
"completions/mean_terminated_length": 872.59375,
|
|
"completions/min_length": 438.0,
|
|
"completions/min_terminated_length": 438.0,
|
|
"entropy": 0.6949086431413889,
|
|
"epoch": 0.2925170068027211,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.322265625,
|
|
"kl": 0.0053970295848557726,
|
|
"learning_rate": 7.080498866213151e-07,
|
|
"loss": -0.0177,
|
|
"num_tokens": 30554973.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 516,
|
|
"step_time": 53.69296963699162
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1371.0,
|
|
"completions/max_terminated_length": 1371.0,
|
|
"completions/mean_length": 798.9375,
|
|
"completions/mean_terminated_length": 798.9375,
|
|
"completions/min_length": 267.0,
|
|
"completions/min_terminated_length": 267.0,
|
|
"entropy": 0.7295571155846119,
|
|
"epoch": 0.29308390022675734,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2158203125,
|
|
"kl": 0.006229313599760644,
|
|
"learning_rate": 7.074829931972789e-07,
|
|
"loss": -0.0047,
|
|
"num_tokens": 30615793.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 517,
|
|
"step_time": 48.61371657811105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1616.0,
|
|
"completions/max_terminated_length": 1616.0,
|
|
"completions/mean_length": 723.1875,
|
|
"completions/mean_terminated_length": 723.1875,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.6038615498691797,
|
|
"epoch": 0.29365079365079366,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.008144882856868207,
|
|
"learning_rate": 7.069160997732426e-07,
|
|
"loss": -0.0154,
|
|
"num_tokens": 30672197.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 518,
|
|
"step_time": 57.058188535273075
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1356.0,
|
|
"completions/max_terminated_length": 1356.0,
|
|
"completions/mean_length": 752.84375,
|
|
"completions/mean_terminated_length": 752.84375,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.9652407057583332,
|
|
"epoch": 0.2942176870748299,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.004668698733439669,
|
|
"learning_rate": 7.063492063492064e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 30729227.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 519,
|
|
"step_time": 46.85251941997558
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1378.0,
|
|
"completions/max_terminated_length": 1378.0,
|
|
"completions/mean_length": 867.515625,
|
|
"completions/mean_terminated_length": 867.515625,
|
|
"completions/min_length": 424.0,
|
|
"completions/min_terminated_length": 424.0,
|
|
"entropy": 0.6276636626571417,
|
|
"epoch": 0.2947845804988662,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.00915596455160994,
|
|
"learning_rate": 7.0578231292517e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 30798044.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7811011672019958,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 520,
|
|
"step_time": 50.15170988161117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1555.0,
|
|
"completions/max_terminated_length": 1555.0,
|
|
"completions/mean_length": 780.3125,
|
|
"completions/mean_terminated_length": 780.3125,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 0.853675838559866,
|
|
"epoch": 0.2953514739229025,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.010571421415079385,
|
|
"learning_rate": 7.052154195011338e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 30860424.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 521,
|
|
"step_time": 54.65656331926584
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1751.0,
|
|
"completions/max_terminated_length": 1751.0,
|
|
"completions/mean_length": 722.125,
|
|
"completions/mean_terminated_length": 722.125,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 0.7592974063009024,
|
|
"epoch": 0.29591836734693877,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.0060547324537765235,
|
|
"learning_rate": 7.046485260770974e-07,
|
|
"loss": -0.0006,
|
|
"num_tokens": 30916824.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 522,
|
|
"step_time": 61.631608521565795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2019.0,
|
|
"completions/max_terminated_length": 2019.0,
|
|
"completions/mean_length": 778.0,
|
|
"completions/mean_terminated_length": 778.0,
|
|
"completions/min_length": 174.0,
|
|
"completions/min_terminated_length": 174.0,
|
|
"entropy": 0.6936698108911514,
|
|
"epoch": 0.29648526077097503,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.400390625,
|
|
"kl": 0.007269517082022503,
|
|
"learning_rate": 7.040816326530612e-07,
|
|
"loss": -0.0056,
|
|
"num_tokens": 30977056.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 523,
|
|
"step_time": 71.95566730946302
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2128.0,
|
|
"completions/max_terminated_length": 2128.0,
|
|
"completions/mean_length": 721.203125,
|
|
"completions/mean_terminated_length": 721.203125,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 0.7360623180866241,
|
|
"epoch": 0.29705215419501135,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.251953125,
|
|
"kl": 0.005839541583554819,
|
|
"learning_rate": 7.035147392290249e-07,
|
|
"loss": 0.0089,
|
|
"num_tokens": 31035229.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 524,
|
|
"step_time": 78.34992097783834
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1264.0,
|
|
"completions/max_terminated_length": 1264.0,
|
|
"completions/mean_length": 735.265625,
|
|
"completions/mean_terminated_length": 735.265625,
|
|
"completions/min_length": 407.0,
|
|
"completions/min_terminated_length": 407.0,
|
|
"entropy": 0.6817584112286568,
|
|
"epoch": 0.2976190476190476,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.007048716477584094,
|
|
"learning_rate": 7.029478458049887e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 31093006.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 525,
|
|
"step_time": 44.232923002913594
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2343.0,
|
|
"completions/max_terminated_length": 2343.0,
|
|
"completions/mean_length": 877.15625,
|
|
"completions/mean_terminated_length": 877.15625,
|
|
"completions/min_length": 432.0,
|
|
"completions/min_terminated_length": 432.0,
|
|
"entropy": 0.848475955426693,
|
|
"epoch": 0.2981859410430839,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.008181585391866975,
|
|
"learning_rate": 7.023809523809523e-07,
|
|
"loss": -0.0175,
|
|
"num_tokens": 31160080.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 526,
|
|
"step_time": 85.97532945871353
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/max_terminated_length": 1376.0,
|
|
"completions/mean_length": 794.890625,
|
|
"completions/mean_terminated_length": 794.890625,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.7595334406942129,
|
|
"epoch": 0.2987528344671202,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.006532763305585831,
|
|
"learning_rate": 7.018140589569161e-07,
|
|
"loss": -0.0185,
|
|
"num_tokens": 31222009.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.53079754114151,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 527,
|
|
"step_time": 48.70655675511807
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2098.0,
|
|
"completions/max_terminated_length": 2098.0,
|
|
"completions/mean_length": 708.890625,
|
|
"completions/mean_terminated_length": 708.890625,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 1.0151412039995193,
|
|
"epoch": 0.29931972789115646,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.012509827109170146,
|
|
"learning_rate": 7.012471655328798e-07,
|
|
"loss": -0.0052,
|
|
"num_tokens": 31280722.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 528,
|
|
"step_time": 76.90691721905023
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2273.0,
|
|
"completions/max_terminated_length": 2273.0,
|
|
"completions/mean_length": 861.96875,
|
|
"completions/mean_terminated_length": 861.96875,
|
|
"completions/min_length": 386.0,
|
|
"completions/min_terminated_length": 386.0,
|
|
"entropy": 0.8241118527948856,
|
|
"epoch": 0.2998866213151927,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.006253472238313407,
|
|
"learning_rate": 7.006802721088436e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 31346664.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 529,
|
|
"step_time": 82.81393055338413
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1587.0,
|
|
"completions/max_terminated_length": 1587.0,
|
|
"completions/mean_length": 736.890625,
|
|
"completions/mean_terminated_length": 736.890625,
|
|
"completions/min_length": 395.0,
|
|
"completions/min_terminated_length": 395.0,
|
|
"entropy": 0.7501304596662521,
|
|
"epoch": 0.30045351473922904,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.006886360570206307,
|
|
"learning_rate": 7.001133786848072e-07,
|
|
"loss": -0.0442,
|
|
"num_tokens": 31405569.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 530,
|
|
"step_time": 55.54344491753727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1377.0,
|
|
"completions/max_terminated_length": 1377.0,
|
|
"completions/mean_length": 768.609375,
|
|
"completions/mean_terminated_length": 768.609375,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 0.8272410221397877,
|
|
"epoch": 0.3010204081632653,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.00595709218760021,
|
|
"learning_rate": 6.995464852607709e-07,
|
|
"loss": -0.0079,
|
|
"num_tokens": 31465880.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.688941478729248,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 531,
|
|
"step_time": 49.46152998972684
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1345.0,
|
|
"completions/mean_length": 765.328125,
|
|
"completions/mean_terminated_length": 712.4603881835938,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.8169015422463417,
|
|
"epoch": 0.30158730158730157,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.013122296615620144,
|
|
"learning_rate": 6.989795918367347e-07,
|
|
"loss": 0.0028,
|
|
"num_tokens": 31526077.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7518165707588196,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 532,
|
|
"step_time": 171.74044344667345
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1168.0,
|
|
"completions/max_terminated_length": 1168.0,
|
|
"completions/mean_length": 706.34375,
|
|
"completions/mean_terminated_length": 706.34375,
|
|
"completions/min_length": 191.0,
|
|
"completions/min_terminated_length": 191.0,
|
|
"entropy": 0.8049690499901772,
|
|
"epoch": 0.3021541950113379,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66015625,
|
|
"kl": 0.014818692725384608,
|
|
"learning_rate": 6.984126984126983e-07,
|
|
"loss": 0.002,
|
|
"num_tokens": 31581315.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.665922224521637,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 533,
|
|
"step_time": 41.171770048327744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1553.0,
|
|
"completions/max_terminated_length": 1553.0,
|
|
"completions/mean_length": 874.203125,
|
|
"completions/mean_terminated_length": 874.203125,
|
|
"completions/min_length": 460.0,
|
|
"completions/min_terminated_length": 460.0,
|
|
"entropy": 0.6727647557854652,
|
|
"epoch": 0.30272108843537415,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.004708681211923249,
|
|
"learning_rate": 6.978458049886621e-07,
|
|
"loss": 0.0222,
|
|
"num_tokens": 31647560.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 534,
|
|
"step_time": 54.527108484879136
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1444.0,
|
|
"completions/max_terminated_length": 1444.0,
|
|
"completions/mean_length": 731.328125,
|
|
"completions/mean_terminated_length": 731.328125,
|
|
"completions/min_length": 70.0,
|
|
"completions/min_terminated_length": 70.0,
|
|
"entropy": 0.6393399555236101,
|
|
"epoch": 0.3032879818594104,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.006850300414953381,
|
|
"learning_rate": 6.972789115646258e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 31704429.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 535,
|
|
"step_time": 51.73478287551552
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1556.0,
|
|
"completions/max_terminated_length": 1556.0,
|
|
"completions/mean_length": 793.09375,
|
|
"completions/mean_terminated_length": 793.09375,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.7901455760002136,
|
|
"epoch": 0.30385487528344673,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.009314977229223587,
|
|
"learning_rate": 6.967120181405896e-07,
|
|
"loss": 0.0375,
|
|
"num_tokens": 31765003.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 536,
|
|
"step_time": 54.62297881022096
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1336.0,
|
|
"completions/max_terminated_length": 1336.0,
|
|
"completions/mean_length": 774.859375,
|
|
"completions/mean_terminated_length": 774.859375,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.8185679968446493,
|
|
"epoch": 0.304421768707483,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.55078125,
|
|
"kl": 0.0047250714560505,
|
|
"learning_rate": 6.961451247165533e-07,
|
|
"loss": 0.0174,
|
|
"num_tokens": 31829394.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6777355074882507,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 537,
|
|
"step_time": 50.495063236914575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1496.0,
|
|
"completions/max_terminated_length": 1496.0,
|
|
"completions/mean_length": 670.0625,
|
|
"completions/mean_terminated_length": 670.0625,
|
|
"completions/min_length": 258.0,
|
|
"completions/min_terminated_length": 258.0,
|
|
"entropy": 0.8335481695830822,
|
|
"epoch": 0.30498866213151926,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5390625,
|
|
"kl": 0.007657179681700654,
|
|
"learning_rate": 6.95578231292517e-07,
|
|
"loss": 0.0083,
|
|
"num_tokens": 31885126.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.528690755367279,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 538,
|
|
"step_time": 53.267095384187996
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1340.0,
|
|
"completions/max_terminated_length": 1340.0,
|
|
"completions/mean_length": 817.046875,
|
|
"completions/mean_terminated_length": 817.046875,
|
|
"completions/min_length": 398.0,
|
|
"completions/min_terminated_length": 398.0,
|
|
"entropy": 0.8791175037622452,
|
|
"epoch": 0.3055555555555556,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.008445959712844342,
|
|
"learning_rate": 6.950113378684806e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 31949209.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 539,
|
|
"step_time": 47.539666498079896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1367.0,
|
|
"completions/max_terminated_length": 1367.0,
|
|
"completions/mean_length": 735.078125,
|
|
"completions/mean_terminated_length": 735.078125,
|
|
"completions/min_length": 380.0,
|
|
"completions/min_terminated_length": 380.0,
|
|
"entropy": 0.6843872237950563,
|
|
"epoch": 0.30612244897959184,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.007761278189718723,
|
|
"learning_rate": 6.944444444444444e-07,
|
|
"loss": -0.0072,
|
|
"num_tokens": 32007294.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 540,
|
|
"step_time": 48.04545336775482
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1537.0,
|
|
"completions/max_terminated_length": 1537.0,
|
|
"completions/mean_length": 729.609375,
|
|
"completions/mean_terminated_length": 729.609375,
|
|
"completions/min_length": 390.0,
|
|
"completions/min_terminated_length": 390.0,
|
|
"entropy": 0.9744435846805573,
|
|
"epoch": 0.3066893424036281,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.009141400048974901,
|
|
"learning_rate": 6.938775510204081e-07,
|
|
"loss": 0.0278,
|
|
"num_tokens": 32064509.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 541,
|
|
"step_time": 53.65542893670499
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1216.0,
|
|
"completions/max_terminated_length": 1216.0,
|
|
"completions/mean_length": 735.3125,
|
|
"completions/mean_terminated_length": 735.3125,
|
|
"completions/min_length": 318.0,
|
|
"completions/min_terminated_length": 318.0,
|
|
"entropy": 0.8175566866993904,
|
|
"epoch": 0.3072562358276644,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.027911689554457553,
|
|
"learning_rate": 6.933106575963719e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 32121881.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 542,
|
|
"step_time": 42.548546058125794
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1617.0,
|
|
"completions/max_terminated_length": 1617.0,
|
|
"completions/mean_length": 668.375,
|
|
"completions/mean_terminated_length": 668.375,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 1.0574189834296703,
|
|
"epoch": 0.3078231292517007,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.010517489456105977,
|
|
"learning_rate": 6.927437641723356e-07,
|
|
"loss": -0.0203,
|
|
"num_tokens": 32175321.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 543,
|
|
"step_time": 58.03311328031123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2352.0,
|
|
"completions/max_terminated_length": 2352.0,
|
|
"completions/mean_length": 815.8125,
|
|
"completions/mean_terminated_length": 815.8125,
|
|
"completions/min_length": 325.0,
|
|
"completions/min_terminated_length": 325.0,
|
|
"entropy": 0.6835882309824228,
|
|
"epoch": 0.30839002267573695,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.337890625,
|
|
"kl": 0.006486616613983642,
|
|
"learning_rate": 6.921768707482994e-07,
|
|
"loss": 0.0123,
|
|
"num_tokens": 32238741.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 544,
|
|
"step_time": 85.0188205158338
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1446.0,
|
|
"completions/max_terminated_length": 1446.0,
|
|
"completions/mean_length": 772.265625,
|
|
"completions/mean_terminated_length": 772.265625,
|
|
"completions/min_length": 342.0,
|
|
"completions/min_terminated_length": 342.0,
|
|
"entropy": 0.7624696642160416,
|
|
"epoch": 0.3089569160997732,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.006251373415580019,
|
|
"learning_rate": 6.91609977324263e-07,
|
|
"loss": -0.0115,
|
|
"num_tokens": 32301606.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 545,
|
|
"step_time": 51.43059713579714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1551.0,
|
|
"completions/max_terminated_length": 1551.0,
|
|
"completions/mean_length": 721.390625,
|
|
"completions/mean_terminated_length": 721.390625,
|
|
"completions/min_length": 195.0,
|
|
"completions/min_terminated_length": 195.0,
|
|
"entropy": 0.8353295251727104,
|
|
"epoch": 0.30952380952380953,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.02856937699834816,
|
|
"learning_rate": 6.910430839002267e-07,
|
|
"loss": 0.0267,
|
|
"num_tokens": 32359799.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 546,
|
|
"step_time": 55.77984538022429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1244.0,
|
|
"completions/max_terminated_length": 1244.0,
|
|
"completions/mean_length": 689.9375,
|
|
"completions/mean_terminated_length": 689.9375,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.6996489334851503,
|
|
"epoch": 0.3100907029478458,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.007028000283753499,
|
|
"learning_rate": 6.904761904761904e-07,
|
|
"loss": -0.0026,
|
|
"num_tokens": 32413123.0,
|
|
"reward": 0.4140625,
|
|
"reward_std": 0.7995766997337341,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 547,
|
|
"step_time": 43.07887785695493
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1278.0,
|
|
"completions/max_terminated_length": 1278.0,
|
|
"completions/mean_length": 832.765625,
|
|
"completions/mean_terminated_length": 832.765625,
|
|
"completions/min_length": 459.0,
|
|
"completions/min_terminated_length": 459.0,
|
|
"entropy": 0.7087801769375801,
|
|
"epoch": 0.31065759637188206,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.007400177259114571,
|
|
"learning_rate": 6.899092970521542e-07,
|
|
"loss": -0.01,
|
|
"num_tokens": 32477244.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 548,
|
|
"step_time": 45.699973184615374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1779.0,
|
|
"completions/max_terminated_length": 1779.0,
|
|
"completions/mean_length": 777.875,
|
|
"completions/mean_terminated_length": 777.875,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 0.7085232455283403,
|
|
"epoch": 0.3112244897959184,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"kl": 0.005462839035317302,
|
|
"learning_rate": 6.893424036281179e-07,
|
|
"loss": 0.011,
|
|
"num_tokens": 32536948.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 549,
|
|
"step_time": 62.400487010367215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1201.0,
|
|
"completions/max_terminated_length": 1201.0,
|
|
"completions/mean_length": 739.65625,
|
|
"completions/mean_terminated_length": 739.65625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.7950146440416574,
|
|
"epoch": 0.31179138321995464,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.65234375,
|
|
"kl": 0.00796415054355748,
|
|
"learning_rate": 6.887755102040817e-07,
|
|
"loss": -0.0046,
|
|
"num_tokens": 32595902.0,
|
|
"reward": 0.515625,
|
|
"reward_std": 0.8402982950210571,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 550,
|
|
"step_time": 42.68447341490537
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1146.0,
|
|
"completions/max_terminated_length": 1146.0,
|
|
"completions/mean_length": 690.515625,
|
|
"completions/mean_terminated_length": 690.515625,
|
|
"completions/min_length": 229.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"entropy": 0.9496628306806087,
|
|
"epoch": 0.3123582766439909,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.013570838025771081,
|
|
"learning_rate": 6.882086167800453e-07,
|
|
"loss": -0.0165,
|
|
"num_tokens": 32650847.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 551,
|
|
"step_time": 40.5395541889593
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1509.0,
|
|
"completions/max_terminated_length": 1509.0,
|
|
"completions/mean_length": 595.09375,
|
|
"completions/mean_terminated_length": 595.09375,
|
|
"completions/min_length": 199.0,
|
|
"completions/min_terminated_length": 199.0,
|
|
"entropy": 0.7972991280257702,
|
|
"epoch": 0.3129251700680272,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.7109375,
|
|
"kl": 0.016013045678846538,
|
|
"learning_rate": 6.87641723356009e-07,
|
|
"loss": -0.025,
|
|
"num_tokens": 32699949.0,
|
|
"reward": 0.5859375,
|
|
"reward_std": 0.9740875959396362,
|
|
"rewards/accuracy_reward_func/mean": 0.53125,
|
|
"rewards/accuracy_reward_func/std": 0.8903138637542725,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 552,
|
|
"step_time": 53.21702896710485
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1199.0,
|
|
"completions/max_terminated_length": 1199.0,
|
|
"completions/mean_length": 733.953125,
|
|
"completions/mean_terminated_length": 733.953125,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.8789013624191284,
|
|
"epoch": 0.3134920634920635,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.01365301915211603,
|
|
"learning_rate": 6.870748299319728e-07,
|
|
"loss": 0.0291,
|
|
"num_tokens": 32756634.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.7983350157737732,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 553,
|
|
"step_time": 42.57348376233131
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1334.0,
|
|
"completions/max_terminated_length": 1334.0,
|
|
"completions/mean_length": 796.890625,
|
|
"completions/mean_terminated_length": 796.890625,
|
|
"completions/min_length": 501.0,
|
|
"completions/min_terminated_length": 501.0,
|
|
"entropy": 0.8650232516229153,
|
|
"epoch": 0.31405895691609975,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.007199519066489302,
|
|
"learning_rate": 6.865079365079364e-07,
|
|
"loss": 0.0163,
|
|
"num_tokens": 32819603.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.503891110420227,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.1171875,
|
|
"rewards/format_reward_func/std": 0.21347814798355103,
|
|
"step": 554,
|
|
"step_time": 47.23435517773032
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1888.0,
|
|
"completions/max_terminated_length": 1888.0,
|
|
"completions/mean_length": 874.203125,
|
|
"completions/mean_terminated_length": 874.203125,
|
|
"completions/min_length": 55.0,
|
|
"completions/min_terminated_length": 55.0,
|
|
"entropy": 0.7912272531539202,
|
|
"epoch": 0.31462585034013607,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.008722388447495177,
|
|
"learning_rate": 6.859410430839002e-07,
|
|
"loss": -0.0152,
|
|
"num_tokens": 32887112.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 555,
|
|
"step_time": 67.5885988837108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1549.0,
|
|
"completions/max_terminated_length": 1549.0,
|
|
"completions/mean_length": 726.765625,
|
|
"completions/mean_terminated_length": 726.765625,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.7201304882764816,
|
|
"epoch": 0.31519274376417233,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.009944985809852369,
|
|
"learning_rate": 6.853741496598639e-07,
|
|
"loss": 0.0282,
|
|
"num_tokens": 32943825.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.7545743584632874,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 556,
|
|
"step_time": 55.162898761220276
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1315.0,
|
|
"completions/max_terminated_length": 1315.0,
|
|
"completions/mean_length": 798.53125,
|
|
"completions/mean_terminated_length": 798.53125,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.7236624136567116,
|
|
"epoch": 0.3157596371882086,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.357421875,
|
|
"kl": 0.006472369088442065,
|
|
"learning_rate": 6.848072562358276e-07,
|
|
"loss": -0.025,
|
|
"num_tokens": 33004995.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 557,
|
|
"step_time": 45.95102402288467
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1390.0,
|
|
"completions/max_terminated_length": 1390.0,
|
|
"completions/mean_length": 719.609375,
|
|
"completions/mean_terminated_length": 719.609375,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 0.7324074916541576,
|
|
"epoch": 0.3163265306122449,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.009824613633099943,
|
|
"learning_rate": 6.842403628117913e-07,
|
|
"loss": -0.0305,
|
|
"num_tokens": 33060794.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 558,
|
|
"step_time": 49.11245018336922
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1244.0,
|
|
"completions/max_terminated_length": 1244.0,
|
|
"completions/mean_length": 722.109375,
|
|
"completions/mean_terminated_length": 722.109375,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.9117118529975414,
|
|
"epoch": 0.3168934240362812,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.006167268380522728,
|
|
"learning_rate": 6.836734693877551e-07,
|
|
"loss": 0.0089,
|
|
"num_tokens": 33117289.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 559,
|
|
"step_time": 44.475634099915624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1211.0,
|
|
"completions/max_terminated_length": 1211.0,
|
|
"completions/mean_length": 752.046875,
|
|
"completions/mean_terminated_length": 752.046875,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.6471206601709127,
|
|
"epoch": 0.31746031746031744,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.016286095182294957,
|
|
"learning_rate": 6.831065759637188e-07,
|
|
"loss": 0.0106,
|
|
"num_tokens": 33176404.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 560,
|
|
"step_time": 43.77733345795423
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1329.0,
|
|
"completions/max_terminated_length": 1329.0,
|
|
"completions/mean_length": 795.78125,
|
|
"completions/mean_terminated_length": 795.78125,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.7221526484936476,
|
|
"epoch": 0.31802721088435376,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.008791988002485596,
|
|
"learning_rate": 6.825396825396826e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 33238966.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7403060793876648,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 561,
|
|
"step_time": 47.99225233308971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1590.0,
|
|
"completions/max_terminated_length": 1590.0,
|
|
"completions/mean_length": 786.9375,
|
|
"completions/mean_terminated_length": 786.9375,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 0.6134519539773464,
|
|
"epoch": 0.31859410430839,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.012130967021221295,
|
|
"learning_rate": 6.819727891156462e-07,
|
|
"loss": 0.0061,
|
|
"num_tokens": 33298642.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 562,
|
|
"step_time": 55.70539809111506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1951.0,
|
|
"completions/max_terminated_length": 1951.0,
|
|
"completions/mean_length": 784.921875,
|
|
"completions/mean_terminated_length": 784.921875,
|
|
"completions/min_length": 124.0,
|
|
"completions/min_terminated_length": 124.0,
|
|
"entropy": 0.8647722918540239,
|
|
"epoch": 0.3191609977324263,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.008675706543726847,
|
|
"learning_rate": 6.8140589569161e-07,
|
|
"loss": 0.0095,
|
|
"num_tokens": 33359685.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 563,
|
|
"step_time": 69.59533071145415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1395.0,
|
|
"completions/max_terminated_length": 1395.0,
|
|
"completions/mean_length": 735.828125,
|
|
"completions/mean_terminated_length": 735.828125,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 0.6789079494774342,
|
|
"epoch": 0.3197278911564626,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"kl": 0.012801257922546938,
|
|
"learning_rate": 6.808390022675736e-07,
|
|
"loss": -0.015,
|
|
"num_tokens": 33418610.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 564,
|
|
"step_time": 50.814760592766106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2015.0,
|
|
"completions/max_terminated_length": 2015.0,
|
|
"completions/mean_length": 674.625,
|
|
"completions/mean_terminated_length": 674.625,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.976968627423048,
|
|
"epoch": 0.32029478458049887,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.62890625,
|
|
"kl": 0.019317736150696874,
|
|
"learning_rate": 6.802721088435374e-07,
|
|
"loss": -0.0228,
|
|
"num_tokens": 33472978.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 565,
|
|
"step_time": 72.5376205034554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1472.0,
|
|
"completions/max_terminated_length": 1472.0,
|
|
"completions/mean_length": 835.484375,
|
|
"completions/mean_terminated_length": 835.484375,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.8818369843065739,
|
|
"epoch": 0.32086167800453513,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.004799109679879621,
|
|
"learning_rate": 6.797052154195011e-07,
|
|
"loss": -0.0068,
|
|
"num_tokens": 33537449.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 566,
|
|
"step_time": 52.162527538836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1780.0,
|
|
"completions/max_terminated_length": 1780.0,
|
|
"completions/mean_length": 813.34375,
|
|
"completions/mean_terminated_length": 813.34375,
|
|
"completions/min_length": 23.0,
|
|
"completions/min_terminated_length": 23.0,
|
|
"entropy": 0.7611702736467123,
|
|
"epoch": 0.32142857142857145,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"kl": 0.00553669105283916,
|
|
"learning_rate": 6.791383219954649e-07,
|
|
"loss": -0.0189,
|
|
"num_tokens": 33600655.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 567,
|
|
"step_time": 63.57557988539338
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1516.0,
|
|
"completions/max_terminated_length": 1516.0,
|
|
"completions/mean_length": 739.171875,
|
|
"completions/mean_terminated_length": 739.171875,
|
|
"completions/min_length": 215.0,
|
|
"completions/min_terminated_length": 215.0,
|
|
"entropy": 0.9445240870118141,
|
|
"epoch": 0.3219954648526077,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.65234375,
|
|
"kl": 0.012653857280383818,
|
|
"learning_rate": 6.785714285714286e-07,
|
|
"loss": -0.0334,
|
|
"num_tokens": 33659810.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7210001945495605,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 568,
|
|
"step_time": 53.3745311005041
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1150.0,
|
|
"completions/max_terminated_length": 1150.0,
|
|
"completions/mean_length": 775.140625,
|
|
"completions/mean_terminated_length": 775.140625,
|
|
"completions/min_length": 264.0,
|
|
"completions/min_terminated_length": 264.0,
|
|
"entropy": 0.903660736978054,
|
|
"epoch": 0.322562358276644,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"kl": 0.004431961293448694,
|
|
"learning_rate": 6.780045351473924e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 33720691.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 569,
|
|
"step_time": 40.83632762823254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1749.0,
|
|
"completions/max_terminated_length": 1749.0,
|
|
"completions/mean_length": 709.078125,
|
|
"completions/mean_terminated_length": 709.078125,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.8605991769582033,
|
|
"epoch": 0.3231292517006803,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.009584751023794524,
|
|
"learning_rate": 6.774376417233559e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 33777448.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.7325600981712341,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 570,
|
|
"step_time": 62.638141446746886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1558.0,
|
|
"completions/max_terminated_length": 1558.0,
|
|
"completions/mean_length": 813.1875,
|
|
"completions/mean_terminated_length": 813.1875,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.761990137398243,
|
|
"epoch": 0.32369614512471656,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.365234375,
|
|
"kl": 0.009937238733982667,
|
|
"learning_rate": 6.768707482993196e-07,
|
|
"loss": -0.0059,
|
|
"num_tokens": 33840436.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 571,
|
|
"step_time": 55.193644834682345
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1181.0,
|
|
"completions/max_terminated_length": 1181.0,
|
|
"completions/mean_length": 753.65625,
|
|
"completions/mean_terminated_length": 753.65625,
|
|
"completions/min_length": 427.0,
|
|
"completions/min_terminated_length": 427.0,
|
|
"entropy": 0.7436486911028624,
|
|
"epoch": 0.3242630385487528,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.63671875,
|
|
"kl": 0.006376277015078813,
|
|
"learning_rate": 6.763038548752834e-07,
|
|
"loss": -0.0119,
|
|
"num_tokens": 33898782.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 572,
|
|
"step_time": 41.69386516418308
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1445.0,
|
|
"completions/max_terminated_length": 1445.0,
|
|
"completions/mean_length": 742.90625,
|
|
"completions/mean_terminated_length": 742.90625,
|
|
"completions/min_length": 361.0,
|
|
"completions/min_terminated_length": 361.0,
|
|
"entropy": 1.085654966533184,
|
|
"epoch": 0.32482993197278914,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.498046875,
|
|
"kl": 0.008307662152219564,
|
|
"learning_rate": 6.757369614512471e-07,
|
|
"loss": -0.0187,
|
|
"num_tokens": 33957776.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 573,
|
|
"step_time": 51.64210286270827
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1926.0,
|
|
"completions/max_terminated_length": 1926.0,
|
|
"completions/mean_length": 825.703125,
|
|
"completions/mean_terminated_length": 825.703125,
|
|
"completions/min_length": 342.0,
|
|
"completions/min_terminated_length": 342.0,
|
|
"entropy": 0.6497799102216959,
|
|
"epoch": 0.3253968253968254,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.009340108736068942,
|
|
"learning_rate": 6.751700680272109e-07,
|
|
"loss": -0.0266,
|
|
"num_tokens": 34020589.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 574,
|
|
"step_time": 67.99144093506038
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2321.0,
|
|
"completions/max_terminated_length": 2321.0,
|
|
"completions/mean_length": 779.375,
|
|
"completions/mean_terminated_length": 779.375,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.9642859268933535,
|
|
"epoch": 0.32596371882086167,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.00579000249854289,
|
|
"learning_rate": 6.746031746031746e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 34083813.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 575,
|
|
"step_time": 86.02688238583505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1810.0,
|
|
"completions/max_terminated_length": 1810.0,
|
|
"completions/mean_length": 765.5,
|
|
"completions/mean_terminated_length": 765.5,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.8294397778809071,
|
|
"epoch": 0.32653061224489793,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.006502250747871585,
|
|
"learning_rate": 6.740362811791383e-07,
|
|
"loss": 0.0436,
|
|
"num_tokens": 34142005.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 576,
|
|
"step_time": 64.16268535330892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1282.0,
|
|
"completions/max_terminated_length": 1282.0,
|
|
"completions/mean_length": 743.984375,
|
|
"completions/mean_terminated_length": 743.984375,
|
|
"completions/min_length": 438.0,
|
|
"completions/min_terminated_length": 438.0,
|
|
"entropy": 0.7624347563832998,
|
|
"epoch": 0.32709750566893425,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.009467750336625613,
|
|
"learning_rate": 6.734693877551019e-07,
|
|
"loss": 0.031,
|
|
"num_tokens": 34200532.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 577,
|
|
"step_time": 45.0961831593886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1390.0,
|
|
"completions/max_terminated_length": 1390.0,
|
|
"completions/mean_length": 729.484375,
|
|
"completions/mean_terminated_length": 729.484375,
|
|
"completions/min_length": 236.0,
|
|
"completions/min_terminated_length": 236.0,
|
|
"entropy": 0.6832804176956415,
|
|
"epoch": 0.3276643990929705,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.671875,
|
|
"kl": 0.010375297861173749,
|
|
"learning_rate": 6.729024943310657e-07,
|
|
"loss": -0.023,
|
|
"num_tokens": 34257195.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 578,
|
|
"step_time": 49.44186826329678
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 984.0,
|
|
"completions/max_terminated_length": 984.0,
|
|
"completions/mean_length": 646.65625,
|
|
"completions/mean_terminated_length": 646.65625,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 0.968621551990509,
|
|
"epoch": 0.3282312925170068,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55078125,
|
|
"kl": 0.006036482838680968,
|
|
"learning_rate": 6.723356009070294e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 34308949.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 579,
|
|
"step_time": 34.739921062253416
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2490.0,
|
|
"completions/max_terminated_length": 2490.0,
|
|
"completions/mean_length": 786.390625,
|
|
"completions/mean_terminated_length": 786.390625,
|
|
"completions/min_length": 274.0,
|
|
"completions/min_terminated_length": 274.0,
|
|
"entropy": 0.6847988367080688,
|
|
"epoch": 0.3287981859410431,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.009381866912008263,
|
|
"learning_rate": 6.717687074829932e-07,
|
|
"loss": 0.0105,
|
|
"num_tokens": 34370286.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7516103982925415,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 580,
|
|
"step_time": 91.41785781644285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1878.0,
|
|
"completions/max_terminated_length": 1878.0,
|
|
"completions/mean_length": 860.703125,
|
|
"completions/mean_terminated_length": 860.703125,
|
|
"completions/min_length": 426.0,
|
|
"completions/min_terminated_length": 426.0,
|
|
"entropy": 0.7797304280102253,
|
|
"epoch": 0.32936507936507936,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.326171875,
|
|
"kl": 0.0077651550964219496,
|
|
"learning_rate": 6.712018140589569e-07,
|
|
"loss": -0.01,
|
|
"num_tokens": 34437251.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 581,
|
|
"step_time": 66.91605470515788
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1384.0,
|
|
"completions/max_terminated_length": 1384.0,
|
|
"completions/mean_length": 736.71875,
|
|
"completions/mean_terminated_length": 736.71875,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.6309777926653624,
|
|
"epoch": 0.3299319727891156,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.01838148111710325,
|
|
"learning_rate": 6.706349206349206e-07,
|
|
"loss": 0.0044,
|
|
"num_tokens": 34494313.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.665922224521637,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 582,
|
|
"step_time": 48.5978308878839
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1467.0,
|
|
"completions/max_terminated_length": 1467.0,
|
|
"completions/mean_length": 819.59375,
|
|
"completions/mean_terminated_length": 819.59375,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.6180352084338665,
|
|
"epoch": 0.33049886621315194,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.006991912028752267,
|
|
"learning_rate": 6.700680272108843e-07,
|
|
"loss": 0.0209,
|
|
"num_tokens": 34557951.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 583,
|
|
"step_time": 51.881614200770855
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1366.0,
|
|
"completions/max_terminated_length": 1366.0,
|
|
"completions/mean_length": 725.734375,
|
|
"completions/mean_terminated_length": 725.734375,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.9028049763292074,
|
|
"epoch": 0.3310657596371882,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.005944862306932919,
|
|
"learning_rate": 6.695011337868481e-07,
|
|
"loss": 0.0046,
|
|
"num_tokens": 34615854.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 584,
|
|
"step_time": 47.98829197511077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1665.0,
|
|
"completions/max_terminated_length": 1665.0,
|
|
"completions/mean_length": 919.8125,
|
|
"completions/mean_terminated_length": 919.8125,
|
|
"completions/min_length": 426.0,
|
|
"completions/min_terminated_length": 426.0,
|
|
"entropy": 0.8066258225589991,
|
|
"epoch": 0.33163265306122447,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.005938148751738481,
|
|
"learning_rate": 6.689342403628118e-07,
|
|
"loss": 0.0004,
|
|
"num_tokens": 34686274.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 585,
|
|
"step_time": 58.45794755406678
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1638.0,
|
|
"completions/max_terminated_length": 1638.0,
|
|
"completions/mean_length": 733.546875,
|
|
"completions/mean_terminated_length": 733.546875,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 0.8840119317173958,
|
|
"epoch": 0.3321995464852608,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.490234375,
|
|
"kl": 0.01541094295680523,
|
|
"learning_rate": 6.683673469387755e-07,
|
|
"loss": -0.017,
|
|
"num_tokens": 34754621.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 586,
|
|
"step_time": 68.61636964324862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/max_terminated_length": 1283.0,
|
|
"completions/mean_length": 785.421875,
|
|
"completions/mean_terminated_length": 785.421875,
|
|
"completions/min_length": 295.0,
|
|
"completions/min_terminated_length": 295.0,
|
|
"entropy": 0.663006603717804,
|
|
"epoch": 0.33276643990929705,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"kl": 0.010400745435617864,
|
|
"learning_rate": 6.678004535147392e-07,
|
|
"loss": 0.0073,
|
|
"num_tokens": 34815840.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 587,
|
|
"step_time": 46.14424010831863
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1618.0,
|
|
"completions/max_terminated_length": 1618.0,
|
|
"completions/mean_length": 720.21875,
|
|
"completions/mean_terminated_length": 720.21875,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.9182410575449467,
|
|
"epoch": 0.3333333333333333,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6875,
|
|
"kl": 0.006522396739455871,
|
|
"learning_rate": 6.672335600907029e-07,
|
|
"loss": -0.0328,
|
|
"num_tokens": 34876782.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5747131109237671,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 588,
|
|
"step_time": 58.905606243759394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1661.0,
|
|
"completions/max_terminated_length": 1661.0,
|
|
"completions/mean_length": 709.96875,
|
|
"completions/mean_terminated_length": 709.96875,
|
|
"completions/min_length": 230.0,
|
|
"completions/min_terminated_length": 230.0,
|
|
"entropy": 0.6486549414694309,
|
|
"epoch": 0.33390022675736963,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.63671875,
|
|
"kl": 0.00879946441273205,
|
|
"learning_rate": 6.666666666666666e-07,
|
|
"loss": -0.0544,
|
|
"num_tokens": 34931516.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6993034482002258,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 589,
|
|
"step_time": 58.1003197664395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1442.0,
|
|
"completions/max_terminated_length": 1442.0,
|
|
"completions/mean_length": 746.8125,
|
|
"completions/mean_terminated_length": 746.8125,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.6505376435816288,
|
|
"epoch": 0.3344671201814059,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.62890625,
|
|
"kl": 0.012137087614974007,
|
|
"learning_rate": 6.660997732426304e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 34989200.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 590,
|
|
"step_time": 50.16918333619833
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1106.0,
|
|
"completions/max_terminated_length": 1106.0,
|
|
"completions/mean_length": 684.921875,
|
|
"completions/mean_terminated_length": 684.921875,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 0.9562710039317608,
|
|
"epoch": 0.33503401360544216,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.0059596063365461305,
|
|
"learning_rate": 6.655328798185941e-07,
|
|
"loss": 0.0091,
|
|
"num_tokens": 35044227.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 591,
|
|
"step_time": 39.63735570758581
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1357.0,
|
|
"completions/max_terminated_length": 1357.0,
|
|
"completions/mean_length": 773.09375,
|
|
"completions/mean_terminated_length": 773.09375,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.9242242239415646,
|
|
"epoch": 0.3356009070294785,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.005730984470574185,
|
|
"learning_rate": 6.649659863945578e-07,
|
|
"loss": 0.0077,
|
|
"num_tokens": 35104305.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 592,
|
|
"step_time": 48.929170694202185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1260.0,
|
|
"completions/max_terminated_length": 1260.0,
|
|
"completions/mean_length": 705.828125,
|
|
"completions/mean_terminated_length": 705.828125,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.9520705752074718,
|
|
"epoch": 0.33616780045351474,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.0053855115111218765,
|
|
"learning_rate": 6.643990929705216e-07,
|
|
"loss": -0.0036,
|
|
"num_tokens": 35159054.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 593,
|
|
"step_time": 44.16639972478151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1430.0,
|
|
"completions/max_terminated_length": 1430.0,
|
|
"completions/mean_length": 751.3125,
|
|
"completions/mean_terminated_length": 751.3125,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 0.7447472456842661,
|
|
"epoch": 0.336734693877551,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.640625,
|
|
"kl": 0.006318980304058641,
|
|
"learning_rate": 6.638321995464852e-07,
|
|
"loss": -0.031,
|
|
"num_tokens": 35215722.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 594,
|
|
"step_time": 50.187391814775765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1541.0,
|
|
"completions/max_terminated_length": 1541.0,
|
|
"completions/mean_length": 815.234375,
|
|
"completions/mean_terminated_length": 815.234375,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.6344550512731075,
|
|
"epoch": 0.3373015873015873,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.007287137719686143,
|
|
"learning_rate": 6.632653061224489e-07,
|
|
"loss": -0.0063,
|
|
"num_tokens": 35277785.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 595,
|
|
"step_time": 54.11244653444737
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1363.0,
|
|
"completions/max_terminated_length": 1363.0,
|
|
"completions/mean_length": 830.734375,
|
|
"completions/mean_terminated_length": 830.734375,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.6941930539906025,
|
|
"epoch": 0.3378684807256236,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3046875,
|
|
"kl": 0.005622691620374098,
|
|
"learning_rate": 6.626984126984126e-07,
|
|
"loss": 0.0111,
|
|
"num_tokens": 35341056.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 596,
|
|
"step_time": 47.846562867984176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1449.0,
|
|
"completions/max_terminated_length": 1449.0,
|
|
"completions/mean_length": 787.59375,
|
|
"completions/mean_terminated_length": 787.59375,
|
|
"completions/min_length": 220.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"entropy": 0.706084543839097,
|
|
"epoch": 0.33843537414965985,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.005095023530884646,
|
|
"learning_rate": 6.621315192743764e-07,
|
|
"loss": -0.0161,
|
|
"num_tokens": 35400366.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7403060793876648,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 597,
|
|
"step_time": 50.668713374994695
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1649.0,
|
|
"completions/max_terminated_length": 1649.0,
|
|
"completions/mean_length": 735.734375,
|
|
"completions/mean_terminated_length": 735.734375,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.7181419394910336,
|
|
"epoch": 0.33900226757369617,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.008124653075356036,
|
|
"learning_rate": 6.615646258503401e-07,
|
|
"loss": 0.0046,
|
|
"num_tokens": 35457917.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 598,
|
|
"step_time": 58.61745241004974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1570.0,
|
|
"completions/max_terminated_length": 1570.0,
|
|
"completions/mean_length": 796.6875,
|
|
"completions/mean_terminated_length": 796.6875,
|
|
"completions/min_length": 369.0,
|
|
"completions/min_terminated_length": 369.0,
|
|
"entropy": 0.7587543912231922,
|
|
"epoch": 0.33956916099773243,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.007574695147923194,
|
|
"learning_rate": 6.609977324263039e-07,
|
|
"loss": -0.0303,
|
|
"num_tokens": 35522113.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 599,
|
|
"step_time": 57.19581332616508
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1587.0,
|
|
"completions/max_terminated_length": 1587.0,
|
|
"completions/mean_length": 836.171875,
|
|
"completions/mean_terminated_length": 836.171875,
|
|
"completions/min_length": 391.0,
|
|
"completions/min_terminated_length": 391.0,
|
|
"entropy": 0.7202887777239084,
|
|
"epoch": 0.3401360544217687,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.455078125,
|
|
"kl": 0.008417851247941144,
|
|
"learning_rate": 6.604308390022676e-07,
|
|
"loss": 0.0094,
|
|
"num_tokens": 35588116.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6023679971694946,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 600,
|
|
"step_time": 57.06244754046202
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1053.0,
|
|
"completions/max_terminated_length": 1053.0,
|
|
"completions/mean_length": 691.84375,
|
|
"completions/mean_terminated_length": 691.84375,
|
|
"completions/min_length": 205.0,
|
|
"completions/min_terminated_length": 205.0,
|
|
"entropy": 0.6493563186377287,
|
|
"epoch": 0.340702947845805,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.62109375,
|
|
"kl": 0.00823523837607354,
|
|
"learning_rate": 6.598639455782312e-07,
|
|
"loss": 0.0209,
|
|
"num_tokens": 35641754.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8003905415534973,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 601,
|
|
"step_time": 38.45988701656461
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1408.0,
|
|
"completions/max_terminated_length": 1408.0,
|
|
"completions/mean_length": 724.703125,
|
|
"completions/mean_terminated_length": 724.703125,
|
|
"completions/min_length": 201.0,
|
|
"completions/min_terminated_length": 201.0,
|
|
"entropy": 0.7356359269469976,
|
|
"epoch": 0.3412698412698413,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.006615829042857513,
|
|
"learning_rate": 6.592970521541949e-07,
|
|
"loss": -0.0006,
|
|
"num_tokens": 35697327.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 602,
|
|
"step_time": 49.652562816627324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1336.0,
|
|
"completions/max_terminated_length": 1336.0,
|
|
"completions/mean_length": 694.265625,
|
|
"completions/mean_terminated_length": 694.265625,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.7202483862638474,
|
|
"epoch": 0.34183673469387754,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6640625,
|
|
"kl": 0.013253367622382939,
|
|
"learning_rate": 6.587301587301587e-07,
|
|
"loss": 0.0548,
|
|
"num_tokens": 35751144.0,
|
|
"reward": 0.4765625,
|
|
"reward_std": 0.8234156966209412,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 603,
|
|
"step_time": 46.8714190647006
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1692.0,
|
|
"completions/max_terminated_length": 1692.0,
|
|
"completions/mean_length": 694.171875,
|
|
"completions/mean_terminated_length": 694.171875,
|
|
"completions/min_length": 294.0,
|
|
"completions/min_terminated_length": 294.0,
|
|
"entropy": 0.9096169024705887,
|
|
"epoch": 0.3424036281179138,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.412109375,
|
|
"kl": 0.009024825121741742,
|
|
"learning_rate": 6.581632653061224e-07,
|
|
"loss": 0.0175,
|
|
"num_tokens": 35806219.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 604,
|
|
"step_time": 59.89671647269279
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2123.0,
|
|
"completions/max_terminated_length": 2123.0,
|
|
"completions/mean_length": 717.921875,
|
|
"completions/mean_terminated_length": 717.921875,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 0.906091034412384,
|
|
"epoch": 0.3429705215419501,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.012036070576868951,
|
|
"learning_rate": 6.575963718820862e-07,
|
|
"loss": 0.0372,
|
|
"num_tokens": 35862350.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5274578928947449,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 605,
|
|
"step_time": 75.91654460690916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1292.0,
|
|
"completions/max_terminated_length": 1292.0,
|
|
"completions/mean_length": 677.90625,
|
|
"completions/mean_terminated_length": 677.90625,
|
|
"completions/min_length": 272.0,
|
|
"completions/min_terminated_length": 272.0,
|
|
"entropy": 0.8916594982147217,
|
|
"epoch": 0.3435374149659864,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.01431399941793643,
|
|
"learning_rate": 6.570294784580499e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 35917184.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 606,
|
|
"step_time": 46.052274979650974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1101.0,
|
|
"completions/max_terminated_length": 1101.0,
|
|
"completions/mean_length": 620.625,
|
|
"completions/mean_terminated_length": 620.625,
|
|
"completions/min_length": 202.0,
|
|
"completions/min_terminated_length": 202.0,
|
|
"entropy": 0.9463461432605982,
|
|
"epoch": 0.34410430839002265,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.640625,
|
|
"kl": 0.012664786409004591,
|
|
"learning_rate": 6.564625850340136e-07,
|
|
"loss": 0.0111,
|
|
"num_tokens": 35966432.0,
|
|
"reward": 0.53125,
|
|
"reward_std": 0.8351171612739563,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 607,
|
|
"step_time": 38.95685858745128
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1611.0,
|
|
"completions/max_terminated_length": 1611.0,
|
|
"completions/mean_length": 765.046875,
|
|
"completions/mean_terminated_length": 765.046875,
|
|
"completions/min_length": 243.0,
|
|
"completions/min_terminated_length": 243.0,
|
|
"entropy": 1.0001036748290062,
|
|
"epoch": 0.34467120181405897,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.466796875,
|
|
"kl": 0.00619954289868474,
|
|
"learning_rate": 6.558956916099773e-07,
|
|
"loss": 0.0105,
|
|
"num_tokens": 36025819.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 608,
|
|
"step_time": 56.710732894949615
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1220.0,
|
|
"completions/max_terminated_length": 1220.0,
|
|
"completions/mean_length": 757.375,
|
|
"completions/mean_terminated_length": 757.375,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.7244377192109823,
|
|
"epoch": 0.34523809523809523,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.01755168184172362,
|
|
"learning_rate": 6.55328798185941e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 36084675.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 609,
|
|
"step_time": 43.188043255358934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2070.0,
|
|
"completions/max_terminated_length": 2070.0,
|
|
"completions/mean_length": 736.578125,
|
|
"completions/mean_terminated_length": 736.578125,
|
|
"completions/min_length": 361.0,
|
|
"completions/min_terminated_length": 361.0,
|
|
"entropy": 0.733847176656127,
|
|
"epoch": 0.3458049886621315,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.0072637570556253195,
|
|
"learning_rate": 6.547619047619047e-07,
|
|
"loss": 0.0248,
|
|
"num_tokens": 36143064.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 610,
|
|
"step_time": 74.52898860815912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1397.0,
|
|
"completions/max_terminated_length": 1397.0,
|
|
"completions/mean_length": 827.15625,
|
|
"completions/mean_terminated_length": 827.15625,
|
|
"completions/min_length": 364.0,
|
|
"completions/min_terminated_length": 364.0,
|
|
"entropy": 0.825311578810215,
|
|
"epoch": 0.3463718820861678,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.007992114246007986,
|
|
"learning_rate": 6.541950113378684e-07,
|
|
"loss": -0.0021,
|
|
"num_tokens": 36207098.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 611,
|
|
"step_time": 49.06635162513703
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1142.0,
|
|
"completions/max_terminated_length": 1142.0,
|
|
"completions/mean_length": 744.046875,
|
|
"completions/mean_terminated_length": 744.046875,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.9102117940783501,
|
|
"epoch": 0.3469387755102041,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.014547953673172742,
|
|
"learning_rate": 6.536281179138322e-07,
|
|
"loss": 0.0291,
|
|
"num_tokens": 36266045.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 612,
|
|
"step_time": 40.136324976570904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1626.0,
|
|
"completions/max_terminated_length": 1626.0,
|
|
"completions/mean_length": 795.96875,
|
|
"completions/mean_terminated_length": 795.96875,
|
|
"completions/min_length": 352.0,
|
|
"completions/min_terminated_length": 352.0,
|
|
"entropy": 0.7217438574880362,
|
|
"epoch": 0.34750566893424034,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.008341065622516908,
|
|
"learning_rate": 6.530612244897958e-07,
|
|
"loss": 0.0119,
|
|
"num_tokens": 36327443.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6412084698677063,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 613,
|
|
"step_time": 57.560481544584036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1208.0,
|
|
"completions/max_terminated_length": 1208.0,
|
|
"completions/mean_length": 699.328125,
|
|
"completions/mean_terminated_length": 699.328125,
|
|
"completions/min_length": 406.0,
|
|
"completions/min_terminated_length": 406.0,
|
|
"entropy": 1.035085018724203,
|
|
"epoch": 0.34807256235827666,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.62890625,
|
|
"kl": 0.008577645698096603,
|
|
"learning_rate": 6.524943310657596e-07,
|
|
"loss": -0.0082,
|
|
"num_tokens": 36382264.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 614,
|
|
"step_time": 42.08609902393073
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1754.0,
|
|
"completions/max_terminated_length": 1754.0,
|
|
"completions/mean_length": 698.40625,
|
|
"completions/mean_terminated_length": 698.40625,
|
|
"completions/min_length": 171.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"entropy": 0.9290982075035572,
|
|
"epoch": 0.3486394557823129,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.011079380084993318,
|
|
"learning_rate": 6.519274376417233e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 36436658.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 615,
|
|
"step_time": 62.292074841447175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1221.0,
|
|
"completions/max_terminated_length": 1221.0,
|
|
"completions/mean_length": 770.203125,
|
|
"completions/mean_terminated_length": 770.203125,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.7892594952136278,
|
|
"epoch": 0.3492063492063492,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.007447789539583027,
|
|
"learning_rate": 6.513605442176871e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 36496631.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 616,
|
|
"step_time": 42.957861450500786
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1806.0,
|
|
"completions/max_terminated_length": 1806.0,
|
|
"completions/mean_length": 881.515625,
|
|
"completions/mean_terminated_length": 881.515625,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.4948860611766577,
|
|
"epoch": 0.3497732426303855,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.005350081744836643,
|
|
"learning_rate": 6.507936507936507e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 36564304.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 617,
|
|
"step_time": 65.9076245026663
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1823.0,
|
|
"completions/max_terminated_length": 1823.0,
|
|
"completions/mean_length": 789.71875,
|
|
"completions/mean_terminated_length": 789.71875,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.7482969025149941,
|
|
"epoch": 0.35034013605442177,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.005938094021985307,
|
|
"learning_rate": 6.502267573696145e-07,
|
|
"loss": 0.0305,
|
|
"num_tokens": 36624910.0,
|
|
"reward": 0.5078125,
|
|
"reward_std": 0.8613950610160828,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 618,
|
|
"step_time": 64.75682531576604
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1336.0,
|
|
"completions/max_terminated_length": 1336.0,
|
|
"completions/mean_length": 816.015625,
|
|
"completions/mean_terminated_length": 816.015625,
|
|
"completions/min_length": 417.0,
|
|
"completions/min_terminated_length": 417.0,
|
|
"entropy": 0.7777763996273279,
|
|
"epoch": 0.35090702947845803,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.006805769502534531,
|
|
"learning_rate": 6.496598639455782e-07,
|
|
"loss": 0.0209,
|
|
"num_tokens": 36689207.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 619,
|
|
"step_time": 47.33159226085991
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1204.0,
|
|
"completions/max_terminated_length": 1204.0,
|
|
"completions/mean_length": 687.671875,
|
|
"completions/mean_terminated_length": 687.671875,
|
|
"completions/min_length": 419.0,
|
|
"completions/min_terminated_length": 419.0,
|
|
"entropy": 0.7947626393288374,
|
|
"epoch": 0.35147392290249435,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.009326640982180834,
|
|
"learning_rate": 6.490929705215419e-07,
|
|
"loss": 0.0291,
|
|
"num_tokens": 36743402.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 620,
|
|
"step_time": 42.9338058270514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1120.0,
|
|
"completions/max_terminated_length": 1120.0,
|
|
"completions/mean_length": 725.171875,
|
|
"completions/mean_terminated_length": 725.171875,
|
|
"completions/min_length": 455.0,
|
|
"completions/min_terminated_length": 455.0,
|
|
"entropy": 0.961219809949398,
|
|
"epoch": 0.3520408163265306,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.412109375,
|
|
"kl": 0.010728677079896443,
|
|
"learning_rate": 6.485260770975056e-07,
|
|
"loss": -0.0033,
|
|
"num_tokens": 36801877.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6784669756889343,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 621,
|
|
"step_time": 40.25142684113234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1513.0,
|
|
"completions/max_terminated_length": 1513.0,
|
|
"completions/mean_length": 756.734375,
|
|
"completions/mean_terminated_length": 756.734375,
|
|
"completions/min_length": 338.0,
|
|
"completions/min_terminated_length": 338.0,
|
|
"entropy": 0.7901823334395885,
|
|
"epoch": 0.3526077097505669,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.01540281844791025,
|
|
"learning_rate": 6.479591836734694e-07,
|
|
"loss": 0.0098,
|
|
"num_tokens": 36860732.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 622,
|
|
"step_time": 53.65702220145613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2213.0,
|
|
"completions/max_terminated_length": 2213.0,
|
|
"completions/mean_length": 769.6875,
|
|
"completions/mean_terminated_length": 769.6875,
|
|
"completions/min_length": 412.0,
|
|
"completions/min_terminated_length": 412.0,
|
|
"entropy": 0.8575314655900002,
|
|
"epoch": 0.3531746031746032,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.31640625,
|
|
"kl": 0.011439471578341909,
|
|
"learning_rate": 6.473922902494331e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 36924144.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 623,
|
|
"step_time": 80.87103377003223
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1911.0,
|
|
"completions/max_terminated_length": 1911.0,
|
|
"completions/mean_length": 895.859375,
|
|
"completions/mean_terminated_length": 895.859375,
|
|
"completions/min_length": 360.0,
|
|
"completions/min_terminated_length": 360.0,
|
|
"entropy": 0.7238175682723522,
|
|
"epoch": 0.35374149659863946,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.009598534583346918,
|
|
"learning_rate": 6.468253968253969e-07,
|
|
"loss": -0.009,
|
|
"num_tokens": 36991183.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.417855441570282,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 624,
|
|
"step_time": 67.44994780607522
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1851.0,
|
|
"completions/max_terminated_length": 1851.0,
|
|
"completions/mean_length": 805.21875,
|
|
"completions/mean_terminated_length": 805.21875,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.8811191525310278,
|
|
"epoch": 0.3543083900226757,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4765625,
|
|
"kl": 0.013165166659746319,
|
|
"learning_rate": 6.462585034013605e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 37053933.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 625,
|
|
"step_time": 65.63539097923785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1300.0,
|
|
"completions/max_terminated_length": 1300.0,
|
|
"completions/mean_length": 837.0,
|
|
"completions/mean_terminated_length": 837.0,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.5979720968753099,
|
|
"epoch": 0.35487528344671204,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.006758156989235431,
|
|
"learning_rate": 6.456916099773242e-07,
|
|
"loss": 0.0063,
|
|
"num_tokens": 37119149.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 626,
|
|
"step_time": 46.38926291838288
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1372.0,
|
|
"completions/max_terminated_length": 1372.0,
|
|
"completions/mean_length": 822.234375,
|
|
"completions/mean_terminated_length": 822.234375,
|
|
"completions/min_length": 438.0,
|
|
"completions/min_terminated_length": 438.0,
|
|
"entropy": 0.8145207818597555,
|
|
"epoch": 0.3554421768707483,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.011722870462108403,
|
|
"learning_rate": 6.451247165532879e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 37183836.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5246290564537048,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 627,
|
|
"step_time": 48.67715365998447
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1306.0,
|
|
"completions/max_terminated_length": 1306.0,
|
|
"completions/mean_length": 733.25,
|
|
"completions/mean_terminated_length": 733.25,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.6681475285440683,
|
|
"epoch": 0.35600907029478457,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.008372545766178519,
|
|
"learning_rate": 6.445578231292517e-07,
|
|
"loss": 0.0385,
|
|
"num_tokens": 37240100.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6542273163795471,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 628,
|
|
"step_time": 46.27730264700949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1167.0,
|
|
"completions/max_terminated_length": 1167.0,
|
|
"completions/mean_length": 795.75,
|
|
"completions/mean_terminated_length": 795.75,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.5886358767747879,
|
|
"epoch": 0.3565759637188209,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.67578125,
|
|
"kl": 0.0060508811147883534,
|
|
"learning_rate": 6.439909297052154e-07,
|
|
"loss": 0.0131,
|
|
"num_tokens": 37300044.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 629,
|
|
"step_time": 41.38639746140689
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1657.0,
|
|
"completions/max_terminated_length": 1657.0,
|
|
"completions/mean_length": 774.125,
|
|
"completions/mean_terminated_length": 774.125,
|
|
"completions/min_length": 387.0,
|
|
"completions/min_terminated_length": 387.0,
|
|
"entropy": 0.6791973039507866,
|
|
"epoch": 0.35714285714285715,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.439453125,
|
|
"kl": 0.008897558262106031,
|
|
"learning_rate": 6.434240362811792e-07,
|
|
"loss": 0.002,
|
|
"num_tokens": 37360996.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 630,
|
|
"step_time": 59.49357104115188
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1532.0,
|
|
"completions/max_terminated_length": 1532.0,
|
|
"completions/mean_length": 799.78125,
|
|
"completions/mean_terminated_length": 799.78125,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.93067716807127,
|
|
"epoch": 0.3577097505668934,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.006074841134250164,
|
|
"learning_rate": 6.428571428571429e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 37422182.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 631,
|
|
"step_time": 54.03444291558117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1553.0,
|
|
"completions/max_terminated_length": 1553.0,
|
|
"completions/mean_length": 642.265625,
|
|
"completions/mean_terminated_length": 642.265625,
|
|
"completions/min_length": 253.0,
|
|
"completions/min_terminated_length": 253.0,
|
|
"entropy": 0.9822626821696758,
|
|
"epoch": 0.35827664399092973,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.012449953996110708,
|
|
"learning_rate": 6.422902494331065e-07,
|
|
"loss": -0.0112,
|
|
"num_tokens": 37473895.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6172134280204773,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 632,
|
|
"step_time": 55.56385569833219
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1296.0,
|
|
"completions/max_terminated_length": 1296.0,
|
|
"completions/mean_length": 765.203125,
|
|
"completions/mean_terminated_length": 765.203125,
|
|
"completions/min_length": 503.0,
|
|
"completions/min_terminated_length": 503.0,
|
|
"entropy": 0.7548550460487604,
|
|
"epoch": 0.358843537414966,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.01114784917444922,
|
|
"learning_rate": 6.417233560090702e-07,
|
|
"loss": -0.0064,
|
|
"num_tokens": 37532668.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6023679971694946,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 633,
|
|
"step_time": 46.07770415302366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1449.0,
|
|
"completions/max_terminated_length": 1449.0,
|
|
"completions/mean_length": 741.890625,
|
|
"completions/mean_terminated_length": 741.890625,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.7171150427311659,
|
|
"epoch": 0.35941043083900226,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.006726305800839327,
|
|
"learning_rate": 6.411564625850339e-07,
|
|
"loss": 0.0264,
|
|
"num_tokens": 37591173.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 634,
|
|
"step_time": 52.218067769892514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1764.0,
|
|
"completions/max_terminated_length": 1764.0,
|
|
"completions/mean_length": 828.9375,
|
|
"completions/mean_terminated_length": 828.9375,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.5765657760202885,
|
|
"epoch": 0.3599773242630385,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.0051835560880135745,
|
|
"learning_rate": 6.405895691609977e-07,
|
|
"loss": -0.0431,
|
|
"num_tokens": 37654033.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7007648944854736,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 635,
|
|
"step_time": 62.01726737897843
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1068.0,
|
|
"completions/max_terminated_length": 1068.0,
|
|
"completions/mean_length": 685.078125,
|
|
"completions/mean_terminated_length": 685.078125,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.7391740418970585,
|
|
"epoch": 0.36054421768707484,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.015797004773048684,
|
|
"learning_rate": 6.400226757369614e-07,
|
|
"loss": -0.0132,
|
|
"num_tokens": 37707358.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 636,
|
|
"step_time": 37.745585405267775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1516.0,
|
|
"completions/mean_length": 814.890625,
|
|
"completions/mean_terminated_length": 762.8095703125,
|
|
"completions/min_length": 379.0,
|
|
"completions/min_terminated_length": 379.0,
|
|
"entropy": 0.8433740437030792,
|
|
"epoch": 0.3611111111111111,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.00606332570896484,
|
|
"learning_rate": 6.394557823129252e-07,
|
|
"loss": 0.0184,
|
|
"num_tokens": 37770343.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 637,
|
|
"step_time": 172.5193542735651
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1524.0,
|
|
"completions/max_terminated_length": 1524.0,
|
|
"completions/mean_length": 809.171875,
|
|
"completions/mean_terminated_length": 809.171875,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.6450223978608847,
|
|
"epoch": 0.36167800453514737,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.010925001610303298,
|
|
"learning_rate": 6.388888888888888e-07,
|
|
"loss": -0.0147,
|
|
"num_tokens": 37831978.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6538955569267273,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 638,
|
|
"step_time": 54.06242776941508
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1231.0,
|
|
"completions/max_terminated_length": 1231.0,
|
|
"completions/mean_length": 794.265625,
|
|
"completions/mean_terminated_length": 794.265625,
|
|
"completions/min_length": 365.0,
|
|
"completions/min_terminated_length": 365.0,
|
|
"entropy": 0.7928502857685089,
|
|
"epoch": 0.3622448979591837,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.330078125,
|
|
"kl": 0.023371351562673226,
|
|
"learning_rate": 6.383219954648526e-07,
|
|
"loss": -0.0102,
|
|
"num_tokens": 37893363.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 639,
|
|
"step_time": 43.288477605208755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1303.0,
|
|
"completions/max_terminated_length": 1303.0,
|
|
"completions/mean_length": 716.71875,
|
|
"completions/mean_terminated_length": 716.71875,
|
|
"completions/min_length": 206.0,
|
|
"completions/min_terminated_length": 206.0,
|
|
"entropy": 0.7845150344073772,
|
|
"epoch": 0.36281179138321995,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.009440813984838314,
|
|
"learning_rate": 6.377551020408163e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 37949521.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 640,
|
|
"step_time": 45.99351770337671
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1177.0,
|
|
"completions/max_terminated_length": 1177.0,
|
|
"completions/mean_length": 654.90625,
|
|
"completions/mean_terminated_length": 654.90625,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.7089138273149729,
|
|
"epoch": 0.3633786848072562,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6171875,
|
|
"kl": 0.010941702232230455,
|
|
"learning_rate": 6.3718820861678e-07,
|
|
"loss": -0.0131,
|
|
"num_tokens": 38003123.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 641,
|
|
"step_time": 42.2809338811785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1807.0,
|
|
"completions/max_terminated_length": 1807.0,
|
|
"completions/mean_length": 812.953125,
|
|
"completions/mean_terminated_length": 812.953125,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.8515927921980619,
|
|
"epoch": 0.36394557823129253,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.007287005981197581,
|
|
"learning_rate": 6.366213151927437e-07,
|
|
"loss": -0.0134,
|
|
"num_tokens": 38066456.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7099072337150574,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 642,
|
|
"step_time": 63.955687711946666
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1562.0,
|
|
"completions/max_terminated_length": 1562.0,
|
|
"completions/mean_length": 778.140625,
|
|
"completions/mean_terminated_length": 778.140625,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.7164397146552801,
|
|
"epoch": 0.3645124716553288,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.39453125,
|
|
"kl": 0.010569749807473272,
|
|
"learning_rate": 6.360544217687075e-07,
|
|
"loss": 0.0153,
|
|
"num_tokens": 38127217.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 643,
|
|
"step_time": 55.2971570501104
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1697.0,
|
|
"completions/max_terminated_length": 1697.0,
|
|
"completions/mean_length": 894.28125,
|
|
"completions/mean_terminated_length": 894.28125,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 0.5776775982230902,
|
|
"epoch": 0.36507936507936506,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.005797176156193018,
|
|
"learning_rate": 6.354875283446711e-07,
|
|
"loss": -0.0145,
|
|
"num_tokens": 38193523.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 644,
|
|
"step_time": 59.91273310314864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1516.0,
|
|
"completions/max_terminated_length": 1516.0,
|
|
"completions/mean_length": 770.0625,
|
|
"completions/mean_terminated_length": 770.0625,
|
|
"completions/min_length": 396.0,
|
|
"completions/min_terminated_length": 396.0,
|
|
"entropy": 0.8040761351585388,
|
|
"epoch": 0.3656462585034014,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.007856981668737717,
|
|
"learning_rate": 6.349206349206349e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 38253775.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 645,
|
|
"step_time": 53.229498567059636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1449.0,
|
|
"completions/max_terminated_length": 1449.0,
|
|
"completions/mean_length": 765.75,
|
|
"completions/mean_terminated_length": 765.75,
|
|
"completions/min_length": 203.0,
|
|
"completions/min_terminated_length": 203.0,
|
|
"entropy": 0.826130636036396,
|
|
"epoch": 0.36621315192743764,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.03074590230244212,
|
|
"learning_rate": 6.343537414965986e-07,
|
|
"loss": -0.0138,
|
|
"num_tokens": 38314383.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 646,
|
|
"step_time": 51.07394890021533
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1198.0,
|
|
"completions/max_terminated_length": 1198.0,
|
|
"completions/mean_length": 705.25,
|
|
"completions/mean_terminated_length": 705.25,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 0.9748920723795891,
|
|
"epoch": 0.3667800453514739,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.009223015804309398,
|
|
"learning_rate": 6.337868480725624e-07,
|
|
"loss": -0.0024,
|
|
"num_tokens": 38372527.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 647,
|
|
"step_time": 44.02167845424265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1439.0,
|
|
"completions/max_terminated_length": 1439.0,
|
|
"completions/mean_length": 750.0,
|
|
"completions/mean_terminated_length": 750.0,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 0.6827063634991646,
|
|
"epoch": 0.3673469387755102,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.034385813371045515,
|
|
"learning_rate": 6.332199546485261e-07,
|
|
"loss": 0.0157,
|
|
"num_tokens": 38434583.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 648,
|
|
"step_time": 51.382030511274934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1735.0,
|
|
"completions/max_terminated_length": 1735.0,
|
|
"completions/mean_length": 732.265625,
|
|
"completions/mean_terminated_length": 732.265625,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.7414722610265017,
|
|
"epoch": 0.3679138321995465,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.007859777222620323,
|
|
"learning_rate": 6.326530612244898e-07,
|
|
"loss": -0.0133,
|
|
"num_tokens": 38492712.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 649,
|
|
"step_time": 62.85209867358208
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 673.828125,
|
|
"completions/mean_terminated_length": 673.828125,
|
|
"completions/min_length": 192.0,
|
|
"completions/min_terminated_length": 192.0,
|
|
"entropy": 0.6841306891292334,
|
|
"epoch": 0.36848072562358275,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.236328125,
|
|
"kl": 0.01202243339503184,
|
|
"learning_rate": 6.320861678004535e-07,
|
|
"loss": 0.0041,
|
|
"num_tokens": 38547941.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 650,
|
|
"step_time": 47.85411164909601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1646.0,
|
|
"completions/max_terminated_length": 1646.0,
|
|
"completions/mean_length": 818.171875,
|
|
"completions/mean_terminated_length": 818.171875,
|
|
"completions/min_length": 286.0,
|
|
"completions/min_terminated_length": 286.0,
|
|
"entropy": 0.8547440283000469,
|
|
"epoch": 0.36904761904761907,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.007522424188209698,
|
|
"learning_rate": 6.315192743764171e-07,
|
|
"loss": 0.0523,
|
|
"num_tokens": 38611616.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.8042545318603516,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 651,
|
|
"step_time": 58.61739876680076
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1575.0,
|
|
"completions/max_terminated_length": 1575.0,
|
|
"completions/mean_length": 776.84375,
|
|
"completions/mean_terminated_length": 776.84375,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.7679075375199318,
|
|
"epoch": 0.36961451247165533,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.013755738385953009,
|
|
"learning_rate": 6.309523809523809e-07,
|
|
"loss": -0.0162,
|
|
"num_tokens": 38670886.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 652,
|
|
"step_time": 55.199841713532805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1147.0,
|
|
"completions/max_terminated_length": 1147.0,
|
|
"completions/mean_length": 646.296875,
|
|
"completions/mean_terminated_length": 646.296875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.8616637103259563,
|
|
"epoch": 0.3701814058956916,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.016826133127324283,
|
|
"learning_rate": 6.303854875283446e-07,
|
|
"loss": 0.0031,
|
|
"num_tokens": 38724105.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8199823498725891,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 653,
|
|
"step_time": 41.71707282681018
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1329.0,
|
|
"completions/max_terminated_length": 1329.0,
|
|
"completions/mean_length": 723.0625,
|
|
"completions/mean_terminated_length": 723.0625,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 0.7345394361764193,
|
|
"epoch": 0.3707482993197279,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.00642925058491528,
|
|
"learning_rate": 6.298185941043084e-07,
|
|
"loss": 0.0099,
|
|
"num_tokens": 38781181.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 654,
|
|
"step_time": 48.112194929271936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1738.0,
|
|
"completions/mean_length": 763.1875,
|
|
"completions/mean_terminated_length": 710.2857666015625,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.6722492016851902,
|
|
"epoch": 0.3713151927437642,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.234375,
|
|
"kl": 0.014458111167186871,
|
|
"learning_rate": 6.292517006802721e-07,
|
|
"loss": 0.0161,
|
|
"num_tokens": 38840721.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 655,
|
|
"step_time": 172.27003202214837
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1468.0,
|
|
"completions/max_terminated_length": 1468.0,
|
|
"completions/mean_length": 760.328125,
|
|
"completions/mean_terminated_length": 760.328125,
|
|
"completions/min_length": 231.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"entropy": 0.932518457993865,
|
|
"epoch": 0.37188208616780044,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.0074783492455026135,
|
|
"learning_rate": 6.286848072562359e-07,
|
|
"loss": -0.0052,
|
|
"num_tokens": 38899718.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8003905415534973,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 656,
|
|
"step_time": 51.769867525435984
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1344.0,
|
|
"completions/max_terminated_length": 1344.0,
|
|
"completions/mean_length": 781.28125,
|
|
"completions/mean_terminated_length": 781.28125,
|
|
"completions/min_length": 469.0,
|
|
"completions/min_terminated_length": 469.0,
|
|
"entropy": 0.9775930866599083,
|
|
"epoch": 0.37244897959183676,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6484375,
|
|
"kl": 0.00942064425908029,
|
|
"learning_rate": 6.281179138321994e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 38960424.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 657,
|
|
"step_time": 47.14356283005327
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1216.0,
|
|
"completions/max_terminated_length": 1216.0,
|
|
"completions/mean_length": 653.53125,
|
|
"completions/mean_terminated_length": 653.53125,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.6461905986070633,
|
|
"epoch": 0.373015873015873,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.70703125,
|
|
"kl": 0.008139535872032866,
|
|
"learning_rate": 6.275510204081632e-07,
|
|
"loss": -0.0092,
|
|
"num_tokens": 39011698.0,
|
|
"reward": 0.5390625,
|
|
"reward_std": 0.8605308532714844,
|
|
"rewards/accuracy_reward_func/mean": 0.5,
|
|
"rewards/accuracy_reward_func/std": 0.8728715777397156,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 658,
|
|
"step_time": 42.68787129409611
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1156.0,
|
|
"completions/max_terminated_length": 1156.0,
|
|
"completions/mean_length": 648.53125,
|
|
"completions/mean_terminated_length": 648.53125,
|
|
"completions/min_length": 162.0,
|
|
"completions/min_terminated_length": 162.0,
|
|
"entropy": 0.8370173312723637,
|
|
"epoch": 0.3735827664399093,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.007988057477632537,
|
|
"learning_rate": 6.269841269841269e-07,
|
|
"loss": -0.011,
|
|
"num_tokens": 39065180.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.745356023311615,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 659,
|
|
"step_time": 42.09308795258403
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2044.0,
|
|
"completions/max_terminated_length": 2044.0,
|
|
"completions/mean_length": 696.015625,
|
|
"completions/mean_terminated_length": 696.015625,
|
|
"completions/min_length": 288.0,
|
|
"completions/min_terminated_length": 288.0,
|
|
"entropy": 0.7783854361623526,
|
|
"epoch": 0.3741496598639456,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.006894381207530387,
|
|
"learning_rate": 6.264172335600907e-07,
|
|
"loss": -0.024,
|
|
"num_tokens": 39119637.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 660,
|
|
"step_time": 73.59489313047379
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1724.0,
|
|
"completions/max_terminated_length": 1724.0,
|
|
"completions/mean_length": 835.203125,
|
|
"completions/mean_terminated_length": 835.203125,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.6195222213864326,
|
|
"epoch": 0.37471655328798187,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.005619698480586521,
|
|
"learning_rate": 6.258503401360544e-07,
|
|
"loss": -0.0191,
|
|
"num_tokens": 39184170.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 661,
|
|
"step_time": 60.97861018963158
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1956.0,
|
|
"completions/max_terminated_length": 1956.0,
|
|
"completions/mean_length": 794.796875,
|
|
"completions/mean_terminated_length": 794.796875,
|
|
"completions/min_length": 300.0,
|
|
"completions/min_terminated_length": 300.0,
|
|
"entropy": 0.7450539339333773,
|
|
"epoch": 0.37528344671201813,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.024208410541177727,
|
|
"learning_rate": 6.252834467120182e-07,
|
|
"loss": 0.0178,
|
|
"num_tokens": 39245533.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 662,
|
|
"step_time": 69.1192515194416
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1511.0,
|
|
"completions/max_terminated_length": 1511.0,
|
|
"completions/mean_length": 846.6875,
|
|
"completions/mean_terminated_length": 846.6875,
|
|
"completions/min_length": 249.0,
|
|
"completions/min_terminated_length": 249.0,
|
|
"entropy": 0.7966278977692127,
|
|
"epoch": 0.3758503401360544,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.008449947970802896,
|
|
"learning_rate": 6.247165532879818e-07,
|
|
"loss": -0.0103,
|
|
"num_tokens": 39310665.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.556738555431366,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 663,
|
|
"step_time": 53.45658629667014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1191.0,
|
|
"completions/max_terminated_length": 1191.0,
|
|
"completions/mean_length": 660.96875,
|
|
"completions/mean_terminated_length": 660.96875,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.8322676233947277,
|
|
"epoch": 0.3764172335600907,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.60546875,
|
|
"kl": 0.007554040043032728,
|
|
"learning_rate": 6.241496598639456e-07,
|
|
"loss": -0.0226,
|
|
"num_tokens": 39362687.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7263124585151672,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 664,
|
|
"step_time": 41.881405444815755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1663.0,
|
|
"completions/max_terminated_length": 1663.0,
|
|
"completions/mean_length": 760.390625,
|
|
"completions/mean_terminated_length": 760.390625,
|
|
"completions/min_length": 267.0,
|
|
"completions/min_terminated_length": 267.0,
|
|
"entropy": 0.7877790946513414,
|
|
"epoch": 0.376984126984127,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.009599128432455473,
|
|
"learning_rate": 6.235827664399092e-07,
|
|
"loss": 0.0109,
|
|
"num_tokens": 39420464.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 665,
|
|
"step_time": 59.588500671088696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1488.0,
|
|
"completions/max_terminated_length": 1488.0,
|
|
"completions/mean_length": 782.390625,
|
|
"completions/mean_terminated_length": 782.390625,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.831054326146841,
|
|
"epoch": 0.37755102040816324,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.008098521488136612,
|
|
"learning_rate": 6.23015873015873e-07,
|
|
"loss": 0.0094,
|
|
"num_tokens": 39483369.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 666,
|
|
"step_time": 52.17609938047826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1439.0,
|
|
"completions/max_terminated_length": 1439.0,
|
|
"completions/mean_length": 687.109375,
|
|
"completions/mean_terminated_length": 687.109375,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 0.6305001396685839,
|
|
"epoch": 0.37811791383219956,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.011480430475785397,
|
|
"learning_rate": 6.224489795918367e-07,
|
|
"loss": -0.0097,
|
|
"num_tokens": 39537720.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6871842741966248,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 667,
|
|
"step_time": 50.57987789902836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1848.0,
|
|
"completions/max_terminated_length": 1848.0,
|
|
"completions/mean_length": 845.109375,
|
|
"completions/mean_terminated_length": 845.109375,
|
|
"completions/min_length": 419.0,
|
|
"completions/min_terminated_length": 419.0,
|
|
"entropy": 0.7953198794275522,
|
|
"epoch": 0.3786848072562358,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.359375,
|
|
"kl": 0.005971537408186123,
|
|
"learning_rate": 6.218820861678005e-07,
|
|
"loss": -0.0096,
|
|
"num_tokens": 39605303.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.53079754114151,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 668,
|
|
"step_time": 67.52485441509634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1423.0,
|
|
"completions/max_terminated_length": 1423.0,
|
|
"completions/mean_length": 783.984375,
|
|
"completions/mean_terminated_length": 783.984375,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 1.113590370863676,
|
|
"epoch": 0.3792517006802721,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.006235114866285585,
|
|
"learning_rate": 6.213151927437641e-07,
|
|
"loss": 0.0144,
|
|
"num_tokens": 39667398.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 669,
|
|
"step_time": 50.54270260501653
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1437.0,
|
|
"completions/max_terminated_length": 1437.0,
|
|
"completions/mean_length": 709.90625,
|
|
"completions/mean_terminated_length": 709.90625,
|
|
"completions/min_length": 210.0,
|
|
"completions/min_terminated_length": 210.0,
|
|
"entropy": 0.7446121480315924,
|
|
"epoch": 0.3798185941043084,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.00550910722813569,
|
|
"learning_rate": 6.207482993197279e-07,
|
|
"loss": 0.0324,
|
|
"num_tokens": 39721312.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 670,
|
|
"step_time": 50.318470236845315
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1349.0,
|
|
"completions/max_terminated_length": 1349.0,
|
|
"completions/mean_length": 669.671875,
|
|
"completions/mean_terminated_length": 669.671875,
|
|
"completions/min_length": 214.0,
|
|
"completions/min_terminated_length": 214.0,
|
|
"entropy": 0.6941411346197128,
|
|
"epoch": 0.38038548752834467,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.010654447032720782,
|
|
"learning_rate": 6.201814058956916e-07,
|
|
"loss": 0.0243,
|
|
"num_tokens": 39777339.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 671,
|
|
"step_time": 50.26656309515238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 657.25,
|
|
"completions/mean_terminated_length": 657.25,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.9057158455252647,
|
|
"epoch": 0.38095238095238093,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.015004386616055854,
|
|
"learning_rate": 6.196145124716554e-07,
|
|
"loss": 0.0005,
|
|
"num_tokens": 39829611.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5092765688896179,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 672,
|
|
"step_time": 47.89571747556329
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1320.0,
|
|
"completions/max_terminated_length": 1320.0,
|
|
"completions/mean_length": 705.796875,
|
|
"completions/mean_terminated_length": 705.796875,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.9398790895938873,
|
|
"epoch": 0.38151927437641725,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.019166790414601564,
|
|
"learning_rate": 6.19047619047619e-07,
|
|
"loss": 0.0112,
|
|
"num_tokens": 39884710.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 673,
|
|
"step_time": 46.67303769942373
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1362.0,
|
|
"completions/max_terminated_length": 1362.0,
|
|
"completions/mean_length": 755.578125,
|
|
"completions/mean_terminated_length": 755.578125,
|
|
"completions/min_length": 361.0,
|
|
"completions/min_terminated_length": 361.0,
|
|
"entropy": 0.775640407577157,
|
|
"epoch": 0.3820861678004535,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4375,
|
|
"kl": 0.005854888193425722,
|
|
"learning_rate": 6.184807256235827e-07,
|
|
"loss": 0.0198,
|
|
"num_tokens": 39944003.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 674,
|
|
"step_time": 49.27024438418448
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1536.0,
|
|
"completions/mean_length": 782.109375,
|
|
"completions/mean_terminated_length": 729.5079956054688,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.7733856309205294,
|
|
"epoch": 0.3826530612244898,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.61328125,
|
|
"kl": 0.007242902065627277,
|
|
"learning_rate": 6.179138321995464e-07,
|
|
"loss": 0.0144,
|
|
"num_tokens": 40004410.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.728528618812561,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 675,
|
|
"step_time": 170.41917849797755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1342.0,
|
|
"completions/max_terminated_length": 1342.0,
|
|
"completions/mean_length": 757.171875,
|
|
"completions/mean_terminated_length": 757.171875,
|
|
"completions/min_length": 330.0,
|
|
"completions/min_terminated_length": 330.0,
|
|
"entropy": 0.8017557859420776,
|
|
"epoch": 0.3832199546485261,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.009273948075133376,
|
|
"learning_rate": 6.173469387755101e-07,
|
|
"loss": -0.0018,
|
|
"num_tokens": 40063125.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 676,
|
|
"step_time": 47.26801723521203
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1510.0,
|
|
"completions/max_terminated_length": 1510.0,
|
|
"completions/mean_length": 698.96875,
|
|
"completions/mean_terminated_length": 698.96875,
|
|
"completions/min_length": 292.0,
|
|
"completions/min_terminated_length": 292.0,
|
|
"entropy": 0.9228569995611906,
|
|
"epoch": 0.38378684807256236,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.384765625,
|
|
"kl": 0.008732386224437505,
|
|
"learning_rate": 6.167800453514739e-07,
|
|
"loss": 0.0308,
|
|
"num_tokens": 40119547.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 677,
|
|
"step_time": 53.07847683969885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 961.0,
|
|
"completions/max_terminated_length": 961.0,
|
|
"completions/mean_length": 564.375,
|
|
"completions/mean_terminated_length": 564.375,
|
|
"completions/min_length": 189.0,
|
|
"completions/min_terminated_length": 189.0,
|
|
"entropy": 1.1396509855985641,
|
|
"epoch": 0.3843537414965986,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.671875,
|
|
"kl": 0.007576363335829228,
|
|
"learning_rate": 6.162131519274376e-07,
|
|
"loss": -0.0141,
|
|
"num_tokens": 40166163.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 678,
|
|
"step_time": 34.3100854344666
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1400.0,
|
|
"completions/max_terminated_length": 1400.0,
|
|
"completions/mean_length": 734.25,
|
|
"completions/mean_terminated_length": 734.25,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.811745923012495,
|
|
"epoch": 0.38492063492063494,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.0064442997390870005,
|
|
"learning_rate": 6.156462585034014e-07,
|
|
"loss": -0.0178,
|
|
"num_tokens": 40229403.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7663560509681702,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 679,
|
|
"step_time": 54.48657435178757
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1274.0,
|
|
"completions/max_terminated_length": 1274.0,
|
|
"completions/mean_length": 687.984375,
|
|
"completions/mean_terminated_length": 687.984375,
|
|
"completions/min_length": 260.0,
|
|
"completions/min_terminated_length": 260.0,
|
|
"entropy": 0.7298913821578026,
|
|
"epoch": 0.3854875283446712,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.63671875,
|
|
"kl": 0.012565153738250956,
|
|
"learning_rate": 6.150793650793651e-07,
|
|
"loss": 0.0173,
|
|
"num_tokens": 40284186.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7841117978096008,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 680,
|
|
"step_time": 45.522766523994505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1611.0,
|
|
"completions/max_terminated_length": 1611.0,
|
|
"completions/mean_length": 838.28125,
|
|
"completions/mean_terminated_length": 838.28125,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.5764037724584341,
|
|
"epoch": 0.38605442176870747,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.00505300224176608,
|
|
"learning_rate": 6.145124716553288e-07,
|
|
"loss": 0.0004,
|
|
"num_tokens": 40347612.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 681,
|
|
"step_time": 56.95555782690644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1423.0,
|
|
"completions/max_terminated_length": 1423.0,
|
|
"completions/mean_length": 707.90625,
|
|
"completions/mean_terminated_length": 707.90625,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 1.054877020418644,
|
|
"epoch": 0.3866213151927438,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.013461493042996153,
|
|
"learning_rate": 6.139455782312924e-07,
|
|
"loss": 0.0265,
|
|
"num_tokens": 40402534.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 682,
|
|
"step_time": 50.564325963146985
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1385.0,
|
|
"completions/max_terminated_length": 1385.0,
|
|
"completions/mean_length": 748.296875,
|
|
"completions/mean_terminated_length": 748.296875,
|
|
"completions/min_length": 300.0,
|
|
"completions/min_terminated_length": 300.0,
|
|
"entropy": 0.686279920861125,
|
|
"epoch": 0.38718820861678005,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.010961992680677213,
|
|
"learning_rate": 6.133786848072562e-07,
|
|
"loss": -0.0266,
|
|
"num_tokens": 40461345.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 683,
|
|
"step_time": 48.89159211330116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1275.0,
|
|
"completions/max_terminated_length": 1275.0,
|
|
"completions/mean_length": 693.125,
|
|
"completions/mean_terminated_length": 693.125,
|
|
"completions/min_length": 330.0,
|
|
"completions/min_terminated_length": 330.0,
|
|
"entropy": 0.8987798392772675,
|
|
"epoch": 0.3877551020408163,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.01087091225781478,
|
|
"learning_rate": 6.128117913832199e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 40517033.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.5890451669692993,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.109375,
|
|
"rewards/format_reward_func/std": 0.2083333432674408,
|
|
"step": 684,
|
|
"step_time": 45.06392130162567
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1603.0,
|
|
"completions/max_terminated_length": 1603.0,
|
|
"completions/mean_length": 843.984375,
|
|
"completions/mean_terminated_length": 843.984375,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.7113766092807055,
|
|
"epoch": 0.38832199546485263,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.009259937069145963,
|
|
"learning_rate": 6.122448979591837e-07,
|
|
"loss": 0.0091,
|
|
"num_tokens": 40580544.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7263124585151672,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 685,
|
|
"step_time": 56.562808574177325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2954.0,
|
|
"completions/max_terminated_length": 2954.0,
|
|
"completions/mean_length": 860.171875,
|
|
"completions/mean_terminated_length": 860.171875,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.6110980063676834,
|
|
"epoch": 0.3888888888888889,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.008090968272881582,
|
|
"learning_rate": 6.116780045351474e-07,
|
|
"loss": 0.0026,
|
|
"num_tokens": 40648035.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 686,
|
|
"step_time": 113.29901125375181
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1174.0,
|
|
"completions/max_terminated_length": 1174.0,
|
|
"completions/mean_length": 739.625,
|
|
"completions/mean_terminated_length": 739.625,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.8477944135665894,
|
|
"epoch": 0.38945578231292516,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.005994448976707645,
|
|
"learning_rate": 6.111111111111112e-07,
|
|
"loss": 0.0174,
|
|
"num_tokens": 40705067.0,
|
|
"reward": 0.421875,
|
|
"reward_std": 0.7777600884437561,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 687,
|
|
"step_time": 41.756474861875176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1111.0,
|
|
"completions/max_terminated_length": 1111.0,
|
|
"completions/mean_length": 571.09375,
|
|
"completions/mean_terminated_length": 571.09375,
|
|
"completions/min_length": 191.0,
|
|
"completions/min_terminated_length": 191.0,
|
|
"entropy": 0.814399428665638,
|
|
"epoch": 0.3900226757369615,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.013807805938995443,
|
|
"learning_rate": 6.105442176870748e-07,
|
|
"loss": 0.0035,
|
|
"num_tokens": 40752633.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 688,
|
|
"step_time": 39.67383046634495
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1359.0,
|
|
"completions/max_terminated_length": 1359.0,
|
|
"completions/mean_length": 729.328125,
|
|
"completions/mean_terminated_length": 729.328125,
|
|
"completions/min_length": 437.0,
|
|
"completions/min_terminated_length": 437.0,
|
|
"entropy": 0.948745608329773,
|
|
"epoch": 0.39058956916099774,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.703125,
|
|
"kl": 0.013188449913286604,
|
|
"learning_rate": 6.099773242630385e-07,
|
|
"loss": 0.0311,
|
|
"num_tokens": 40809830.0,
|
|
"reward": 0.515625,
|
|
"reward_std": 0.8772646188735962,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 689,
|
|
"step_time": 47.61505999881774
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1398.0,
|
|
"completions/max_terminated_length": 1398.0,
|
|
"completions/mean_length": 714.296875,
|
|
"completions/mean_terminated_length": 714.296875,
|
|
"completions/min_length": 260.0,
|
|
"completions/min_terminated_length": 260.0,
|
|
"entropy": 0.6849742606282234,
|
|
"epoch": 0.391156462585034,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.013025412554270588,
|
|
"learning_rate": 6.094104308390022e-07,
|
|
"loss": 0.0239,
|
|
"num_tokens": 40865577.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 690,
|
|
"step_time": 48.928758626803756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2890.0,
|
|
"completions/max_terminated_length": 2890.0,
|
|
"completions/mean_length": 752.125,
|
|
"completions/mean_terminated_length": 752.125,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.7944428026676178,
|
|
"epoch": 0.39172335600907027,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.015944800994475372,
|
|
"learning_rate": 6.08843537414966e-07,
|
|
"loss": 0.0012,
|
|
"num_tokens": 40923945.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.417855441570282,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 691,
|
|
"step_time": 107.91727275867015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1478.0,
|
|
"completions/max_terminated_length": 1478.0,
|
|
"completions/mean_length": 745.546875,
|
|
"completions/mean_terminated_length": 745.546875,
|
|
"completions/min_length": 386.0,
|
|
"completions/min_terminated_length": 386.0,
|
|
"entropy": 0.7080104593187571,
|
|
"epoch": 0.3922902494331066,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.01081945282930974,
|
|
"learning_rate": 6.082766439909297e-07,
|
|
"loss": -0.0066,
|
|
"num_tokens": 40982052.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5107961297035217,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 692,
|
|
"step_time": 52.60903904493898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1274.0,
|
|
"completions/max_terminated_length": 1274.0,
|
|
"completions/mean_length": 664.296875,
|
|
"completions/mean_terminated_length": 664.296875,
|
|
"completions/min_length": 244.0,
|
|
"completions/min_terminated_length": 244.0,
|
|
"entropy": 0.9997816383838654,
|
|
"epoch": 0.39285714285714285,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.01066761146648787,
|
|
"learning_rate": 6.077097505668934e-07,
|
|
"loss": 0.014,
|
|
"num_tokens": 41033927.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.7233612537384033,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 693,
|
|
"step_time": 44.80714107491076
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1190.0,
|
|
"completions/max_terminated_length": 1190.0,
|
|
"completions/mean_length": 751.71875,
|
|
"completions/mean_terminated_length": 751.71875,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 1.013720091432333,
|
|
"epoch": 0.3934240362811791,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.009110613173106685,
|
|
"learning_rate": 6.071428571428571e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 41092381.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.8344857692718506,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 694,
|
|
"step_time": 42.28477763570845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1784.0,
|
|
"completions/max_terminated_length": 1784.0,
|
|
"completions/mean_length": 757.359375,
|
|
"completions/mean_terminated_length": 757.359375,
|
|
"completions/min_length": 171.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"entropy": 0.84516691416502,
|
|
"epoch": 0.39399092970521543,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.09765625,
|
|
"kl": 0.041434024315094575,
|
|
"learning_rate": 6.065759637188208e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 41153052.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.1666666716337204,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 695,
|
|
"step_time": 63.974081702530384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1708.0,
|
|
"completions/max_terminated_length": 1708.0,
|
|
"completions/mean_length": 778.671875,
|
|
"completions/mean_terminated_length": 778.671875,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.6577491778880358,
|
|
"epoch": 0.3945578231292517,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.007940168958157301,
|
|
"learning_rate": 6.060090702947846e-07,
|
|
"loss": -0.005,
|
|
"num_tokens": 41214959.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7614454030990601,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 696,
|
|
"step_time": 62.58700111228973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1214.0,
|
|
"completions/max_terminated_length": 1214.0,
|
|
"completions/mean_length": 682.875,
|
|
"completions/mean_terminated_length": 682.875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.8445435054600239,
|
|
"epoch": 0.39512471655328796,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.007366003163042478,
|
|
"learning_rate": 6.054421768707482e-07,
|
|
"loss": -0.0153,
|
|
"num_tokens": 41269151.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 697,
|
|
"step_time": 42.70040045212954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1299.0,
|
|
"completions/max_terminated_length": 1299.0,
|
|
"completions/mean_length": 734.59375,
|
|
"completions/mean_terminated_length": 734.59375,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.8305799197405577,
|
|
"epoch": 0.3956916099773243,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.016614578402368352,
|
|
"learning_rate": 6.04875283446712e-07,
|
|
"loss": -0.0005,
|
|
"num_tokens": 41326429.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 698,
|
|
"step_time": 46.312332388944924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1441.0,
|
|
"completions/max_terminated_length": 1441.0,
|
|
"completions/mean_length": 692.390625,
|
|
"completions/mean_terminated_length": 692.390625,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.744658574461937,
|
|
"epoch": 0.39625850340136054,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.263671875,
|
|
"kl": 0.025707505556056276,
|
|
"learning_rate": 6.043083900226757e-07,
|
|
"loss": -0.0077,
|
|
"num_tokens": 41382806.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 699,
|
|
"step_time": 50.56061425153166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1816.0,
|
|
"completions/max_terminated_length": 1816.0,
|
|
"completions/mean_length": 721.171875,
|
|
"completions/mean_terminated_length": 721.171875,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.8119492139667273,
|
|
"epoch": 0.3968253968253968,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.00942586176097393,
|
|
"learning_rate": 6.037414965986394e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 41439313.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 700,
|
|
"step_time": 64.30932085122913
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1294.0,
|
|
"completions/max_terminated_length": 1294.0,
|
|
"completions/mean_length": 778.453125,
|
|
"completions/mean_terminated_length": 778.453125,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.6587596759200096,
|
|
"epoch": 0.3973922902494331,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.008683416119311005,
|
|
"learning_rate": 6.031746031746031e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 41500302.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6172134280204773,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 701,
|
|
"step_time": 45.86473724339157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1466.0,
|
|
"completions/max_terminated_length": 1466.0,
|
|
"completions/mean_length": 719.453125,
|
|
"completions/mean_terminated_length": 719.453125,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 0.8286863304674625,
|
|
"epoch": 0.3979591836734694,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.011126842553494498,
|
|
"learning_rate": 6.026077097505669e-07,
|
|
"loss": 0.0188,
|
|
"num_tokens": 41558107.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 702,
|
|
"step_time": 52.28511210810393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1700.0,
|
|
"completions/max_terminated_length": 1700.0,
|
|
"completions/mean_length": 697.484375,
|
|
"completions/mean_terminated_length": 697.484375,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.6992883961647749,
|
|
"epoch": 0.39852607709750565,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"kl": 0.013945808750577271,
|
|
"learning_rate": 6.020408163265306e-07,
|
|
"loss": 0.0323,
|
|
"num_tokens": 41614698.0,
|
|
"reward": 0.46875,
|
|
"reward_std": 0.7961963415145874,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 703,
|
|
"step_time": 60.479034840129316
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1282.0,
|
|
"completions/max_terminated_length": 1282.0,
|
|
"completions/mean_length": 714.59375,
|
|
"completions/mean_terminated_length": 714.59375,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.7452113330364227,
|
|
"epoch": 0.39909297052154197,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.015139883413212374,
|
|
"learning_rate": 6.014739229024944e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 41671320.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 704,
|
|
"step_time": 45.73909159749746
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1097.0,
|
|
"completions/max_terminated_length": 1097.0,
|
|
"completions/mean_length": 676.25,
|
|
"completions/mean_terminated_length": 676.25,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.7598608788102865,
|
|
"epoch": 0.39965986394557823,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.011453291866928339,
|
|
"learning_rate": 6.00907029478458e-07,
|
|
"loss": 0.0065,
|
|
"num_tokens": 41724608.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 705,
|
|
"step_time": 38.95879540685564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1437.0,
|
|
"completions/max_terminated_length": 1437.0,
|
|
"completions/mean_length": 739.6875,
|
|
"completions/mean_terminated_length": 739.6875,
|
|
"completions/min_length": 180.0,
|
|
"completions/min_terminated_length": 180.0,
|
|
"entropy": 0.9063232727348804,
|
|
"epoch": 0.4002267573696145,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.010939083382254466,
|
|
"learning_rate": 6.003401360544217e-07,
|
|
"loss": -0.0217,
|
|
"num_tokens": 41784140.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6422230005264282,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 706,
|
|
"step_time": 51.03928552661091
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1576.0,
|
|
"completions/max_terminated_length": 1576.0,
|
|
"completions/mean_length": 834.78125,
|
|
"completions/mean_terminated_length": 834.78125,
|
|
"completions/min_length": 488.0,
|
|
"completions/min_terminated_length": 488.0,
|
|
"entropy": 0.6948952544480562,
|
|
"epoch": 0.4007936507936508,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.005924797136685811,
|
|
"learning_rate": 5.997732426303854e-07,
|
|
"loss": 0.0205,
|
|
"num_tokens": 41848046.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 707,
|
|
"step_time": 55.708213401027024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1443.0,
|
|
"completions/max_terminated_length": 1443.0,
|
|
"completions/mean_length": 765.078125,
|
|
"completions/mean_terminated_length": 765.078125,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.7151809260249138,
|
|
"epoch": 0.4013605442176871,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4375,
|
|
"kl": 0.005187169241253287,
|
|
"learning_rate": 5.992063492063492e-07,
|
|
"loss": 0.016,
|
|
"num_tokens": 41906531.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 708,
|
|
"step_time": 50.46130122616887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1518.0,
|
|
"completions/max_terminated_length": 1518.0,
|
|
"completions/mean_length": 716.25,
|
|
"completions/mean_terminated_length": 716.25,
|
|
"completions/min_length": 258.0,
|
|
"completions/min_terminated_length": 258.0,
|
|
"entropy": 0.7209887355566025,
|
|
"epoch": 0.40192743764172334,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.60546875,
|
|
"kl": 0.013952078035799786,
|
|
"learning_rate": 5.986394557823129e-07,
|
|
"loss": -0.045,
|
|
"num_tokens": 41963499.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 709,
|
|
"step_time": 53.49163555726409
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1413.0,
|
|
"completions/max_terminated_length": 1413.0,
|
|
"completions/mean_length": 783.625,
|
|
"completions/mean_terminated_length": 783.625,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.8843955546617508,
|
|
"epoch": 0.40249433106575966,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.005734575490350835,
|
|
"learning_rate": 5.980725623582767e-07,
|
|
"loss": -0.0007,
|
|
"num_tokens": 42023875.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 710,
|
|
"step_time": 50.09232422430068
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2000.0,
|
|
"completions/max_terminated_length": 2000.0,
|
|
"completions/mean_length": 945.59375,
|
|
"completions/mean_terminated_length": 945.59375,
|
|
"completions/min_length": 446.0,
|
|
"completions/min_terminated_length": 446.0,
|
|
"entropy": 0.6307933144271374,
|
|
"epoch": 0.4030612244897959,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.006169469590531662,
|
|
"learning_rate": 5.975056689342404e-07,
|
|
"loss": -0.0265,
|
|
"num_tokens": 42093881.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.8902789950370789,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 711,
|
|
"step_time": 71.84906288422644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1299.0,
|
|
"completions/max_terminated_length": 1299.0,
|
|
"completions/mean_length": 707.390625,
|
|
"completions/mean_terminated_length": 707.390625,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 1.027089785784483,
|
|
"epoch": 0.4036281179138322,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.0093746821366949,
|
|
"learning_rate": 5.969387755102042e-07,
|
|
"loss": 0.0229,
|
|
"num_tokens": 42150370.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 712,
|
|
"step_time": 45.99393534846604
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1565.0,
|
|
"completions/max_terminated_length": 1565.0,
|
|
"completions/mean_length": 784.09375,
|
|
"completions/mean_terminated_length": 784.09375,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.7007075678557158,
|
|
"epoch": 0.4041950113378685,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.011935453439946286,
|
|
"learning_rate": 5.963718820861677e-07,
|
|
"loss": -0.0315,
|
|
"num_tokens": 42211264.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.09375,
|
|
"rewards/format_reward_func/std": 0.19669894874095917,
|
|
"step": 713,
|
|
"step_time": 55.61997680179775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1262.0,
|
|
"completions/max_terminated_length": 1262.0,
|
|
"completions/mean_length": 734.59375,
|
|
"completions/mean_terminated_length": 734.59375,
|
|
"completions/min_length": 205.0,
|
|
"completions/min_terminated_length": 205.0,
|
|
"entropy": 0.807158350944519,
|
|
"epoch": 0.40476190476190477,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.014308157653431408,
|
|
"learning_rate": 5.958049886621314e-07,
|
|
"loss": 0.0274,
|
|
"num_tokens": 42270846.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 714,
|
|
"step_time": 46.46322725806385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2698.0,
|
|
"completions/max_terminated_length": 2698.0,
|
|
"completions/mean_length": 774.796875,
|
|
"completions/mean_terminated_length": 774.796875,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 0.656152818351984,
|
|
"epoch": 0.40532879818594103,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.009226910246070474,
|
|
"learning_rate": 5.952380952380952e-07,
|
|
"loss": 0.029,
|
|
"num_tokens": 42329793.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7927232980728149,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 715,
|
|
"step_time": 99.79910800233483
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1366.0,
|
|
"completions/max_terminated_length": 1366.0,
|
|
"completions/mean_length": 810.203125,
|
|
"completions/mean_terminated_length": 810.203125,
|
|
"completions/min_length": 455.0,
|
|
"completions/min_terminated_length": 455.0,
|
|
"entropy": 0.8198779430240393,
|
|
"epoch": 0.40589569160997735,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.00660677463747561,
|
|
"learning_rate": 5.946712018140589e-07,
|
|
"loss": 0.0181,
|
|
"num_tokens": 42390966.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 716,
|
|
"step_time": 48.02453545387834
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1277.0,
|
|
"completions/max_terminated_length": 1277.0,
|
|
"completions/mean_length": 706.90625,
|
|
"completions/mean_terminated_length": 706.90625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.9216723181307316,
|
|
"epoch": 0.4064625850340136,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.007653110675164498,
|
|
"learning_rate": 5.941043083900227e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 42447512.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 717,
|
|
"step_time": 45.191788436844945
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1568.0,
|
|
"completions/max_terminated_length": 1568.0,
|
|
"completions/mean_length": 790.5625,
|
|
"completions/mean_terminated_length": 790.5625,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.6126455087214708,
|
|
"epoch": 0.4070294784580499,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.011935973423533142,
|
|
"learning_rate": 5.935374149659864e-07,
|
|
"loss": 0.0113,
|
|
"num_tokens": 42507436.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 718,
|
|
"step_time": 55.15782236587256
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1184.0,
|
|
"completions/max_terminated_length": 1184.0,
|
|
"completions/mean_length": 664.59375,
|
|
"completions/mean_terminated_length": 664.59375,
|
|
"completions/min_length": 292.0,
|
|
"completions/min_terminated_length": 292.0,
|
|
"entropy": 0.7403056938201189,
|
|
"epoch": 0.4075963718820862,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.00788785464828834,
|
|
"learning_rate": 5.929705215419501e-07,
|
|
"loss": -0.0228,
|
|
"num_tokens": 42560706.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8003905415534973,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 719,
|
|
"step_time": 42.083308012224734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1397.0,
|
|
"completions/max_terminated_length": 1397.0,
|
|
"completions/mean_length": 758.03125,
|
|
"completions/mean_terminated_length": 758.03125,
|
|
"completions/min_length": 398.0,
|
|
"completions/min_terminated_length": 398.0,
|
|
"entropy": 0.7515831179916859,
|
|
"epoch": 0.40816326530612246,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4765625,
|
|
"kl": 0.008381221603485756,
|
|
"learning_rate": 5.924036281179138e-07,
|
|
"loss": 0.0171,
|
|
"num_tokens": 42623868.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7403060793876648,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 720,
|
|
"step_time": 49.98040152899921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1847.0,
|
|
"completions/max_terminated_length": 1847.0,
|
|
"completions/mean_length": 839.5,
|
|
"completions/mean_terminated_length": 839.5,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.5872164405882359,
|
|
"epoch": 0.4087301587301587,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.012213960901135579,
|
|
"learning_rate": 5.918367346938775e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 42687524.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 721,
|
|
"step_time": 65.39091375470161
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1121.0,
|
|
"completions/max_terminated_length": 1121.0,
|
|
"completions/mean_length": 709.765625,
|
|
"completions/mean_terminated_length": 709.765625,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.7878752686083317,
|
|
"epoch": 0.409297052154195,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.011058046336984262,
|
|
"learning_rate": 5.912698412698412e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 42744709.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 722,
|
|
"step_time": 41.04239758849144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1357.0,
|
|
"completions/max_terminated_length": 1357.0,
|
|
"completions/mean_length": 706.921875,
|
|
"completions/mean_terminated_length": 706.921875,
|
|
"completions/min_length": 239.0,
|
|
"completions/min_terminated_length": 239.0,
|
|
"entropy": 0.8756008334457874,
|
|
"epoch": 0.4098639455782313,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"kl": 0.008593476144596934,
|
|
"learning_rate": 5.90702947845805e-07,
|
|
"loss": 0.016,
|
|
"num_tokens": 42807376.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 723,
|
|
"step_time": 51.893509834073484
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1289.0,
|
|
"completions/mean_length": 837.21875,
|
|
"completions/mean_terminated_length": 785.4921264648438,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.8207143619656563,
|
|
"epoch": 0.41043083900226757,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.00925534228736069,
|
|
"learning_rate": 5.901360544217687e-07,
|
|
"loss": 0.0337,
|
|
"num_tokens": 42871790.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 724,
|
|
"step_time": 171.53516479954123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1429.0,
|
|
"completions/max_terminated_length": 1429.0,
|
|
"completions/mean_length": 804.09375,
|
|
"completions/mean_terminated_length": 804.09375,
|
|
"completions/min_length": 213.0,
|
|
"completions/min_terminated_length": 213.0,
|
|
"entropy": 0.7975950799882412,
|
|
"epoch": 0.41099773242630383,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.009272914787288755,
|
|
"learning_rate": 5.895691609977324e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 42935236.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 725,
|
|
"step_time": 51.1166734918952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1159.0,
|
|
"completions/max_terminated_length": 1159.0,
|
|
"completions/mean_length": 708.921875,
|
|
"completions/mean_terminated_length": 708.921875,
|
|
"completions/min_length": 271.0,
|
|
"completions/min_terminated_length": 271.0,
|
|
"entropy": 0.7010960951447487,
|
|
"epoch": 0.41156462585034015,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.60546875,
|
|
"kl": 0.008177276497008279,
|
|
"learning_rate": 5.890022675736961e-07,
|
|
"loss": -0.0094,
|
|
"num_tokens": 42990887.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.8110105395317078,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 726,
|
|
"step_time": 41.16157591063529
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1847.0,
|
|
"completions/max_terminated_length": 1847.0,
|
|
"completions/mean_length": 719.390625,
|
|
"completions/mean_terminated_length": 719.390625,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.8196245543658733,
|
|
"epoch": 0.4121315192743764,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.013137170026311651,
|
|
"learning_rate": 5.884353741496599e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 43046920.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 727,
|
|
"step_time": 65.44489248842001
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1305.0,
|
|
"completions/max_terminated_length": 1305.0,
|
|
"completions/mean_length": 674.890625,
|
|
"completions/mean_terminated_length": 674.890625,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 1.0484230630099773,
|
|
"epoch": 0.4126984126984127,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.008125306529109366,
|
|
"learning_rate": 5.878684807256236e-07,
|
|
"loss": -0.0062,
|
|
"num_tokens": 43102601.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5092765688896179,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 728,
|
|
"step_time": 46.45828823093325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1484.0,
|
|
"completions/max_terminated_length": 1484.0,
|
|
"completions/mean_length": 743.734375,
|
|
"completions/mean_terminated_length": 743.734375,
|
|
"completions/min_length": 234.0,
|
|
"completions/min_terminated_length": 234.0,
|
|
"entropy": 0.8930370584130287,
|
|
"epoch": 0.413265306122449,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.412109375,
|
|
"kl": 0.01043572741036769,
|
|
"learning_rate": 5.873015873015873e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 43163744.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 729,
|
|
"step_time": 54.263520089909434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1262.0,
|
|
"completions/max_terminated_length": 1262.0,
|
|
"completions/mean_length": 707.625,
|
|
"completions/mean_terminated_length": 707.625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.8807930462062359,
|
|
"epoch": 0.41383219954648526,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.006125825515482575,
|
|
"learning_rate": 5.86734693877551e-07,
|
|
"loss": 0.0123,
|
|
"num_tokens": 43218936.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 730,
|
|
"step_time": 44.564856003038585
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1719.0,
|
|
"completions/max_terminated_length": 1719.0,
|
|
"completions/mean_length": 715.40625,
|
|
"completions/mean_terminated_length": 715.40625,
|
|
"completions/min_length": 282.0,
|
|
"completions/min_terminated_length": 282.0,
|
|
"entropy": 0.6654192712157965,
|
|
"epoch": 0.4143990929705215,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.66015625,
|
|
"kl": 0.007447896467056125,
|
|
"learning_rate": 5.861678004535146e-07,
|
|
"loss": 0.043,
|
|
"num_tokens": 43275410.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8188852071762085,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 731,
|
|
"step_time": 61.41636692080647
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1438.0,
|
|
"completions/max_terminated_length": 1438.0,
|
|
"completions/mean_length": 732.140625,
|
|
"completions/mean_terminated_length": 732.140625,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.631422147154808,
|
|
"epoch": 0.41496598639455784,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.7578125,
|
|
"kl": 0.008761520963162184,
|
|
"learning_rate": 5.856009070294784e-07,
|
|
"loss": 0.0465,
|
|
"num_tokens": 43333819.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8188852071762085,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 732,
|
|
"step_time": 50.89566903561354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1203.0,
|
|
"completions/max_terminated_length": 1203.0,
|
|
"completions/mean_length": 716.171875,
|
|
"completions/mean_terminated_length": 716.171875,
|
|
"completions/min_length": 215.0,
|
|
"completions/min_terminated_length": 215.0,
|
|
"entropy": 1.1561571098864079,
|
|
"epoch": 0.4155328798185941,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.01393100174027495,
|
|
"learning_rate": 5.850340136054421e-07,
|
|
"loss": 0.0084,
|
|
"num_tokens": 43391030.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.6267831921577454,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 733,
|
|
"step_time": 42.88149905297905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1304.0,
|
|
"completions/max_terminated_length": 1304.0,
|
|
"completions/mean_length": 698.921875,
|
|
"completions/mean_terminated_length": 698.921875,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.8924953229725361,
|
|
"epoch": 0.41609977324263037,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.009557745564961806,
|
|
"learning_rate": 5.844671201814059e-07,
|
|
"loss": 0.0043,
|
|
"num_tokens": 43446617.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 734,
|
|
"step_time": 46.1087064538151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1468.0,
|
|
"completions/max_terminated_length": 1468.0,
|
|
"completions/mean_length": 815.21875,
|
|
"completions/mean_terminated_length": 815.21875,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.82574812322855,
|
|
"epoch": 0.4166666666666667,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.375,
|
|
"kl": 0.011717286106431857,
|
|
"learning_rate": 5.839002267573696e-07,
|
|
"loss": -0.0091,
|
|
"num_tokens": 43508415.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 735,
|
|
"step_time": 51.29634570237249
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1312.0,
|
|
"completions/max_terminated_length": 1312.0,
|
|
"completions/mean_length": 793.21875,
|
|
"completions/mean_terminated_length": 793.21875,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 0.770227137953043,
|
|
"epoch": 0.41723356009070295,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.015240149485180154,
|
|
"learning_rate": 5.833333333333334e-07,
|
|
"loss": -0.0223,
|
|
"num_tokens": 43570285.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 736,
|
|
"step_time": 46.78364960197359
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1126.0,
|
|
"completions/max_terminated_length": 1126.0,
|
|
"completions/mean_length": 765.109375,
|
|
"completions/mean_terminated_length": 765.109375,
|
|
"completions/min_length": 436.0,
|
|
"completions/min_terminated_length": 436.0,
|
|
"entropy": 0.80656623467803,
|
|
"epoch": 0.4178004535147392,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.365234375,
|
|
"kl": 0.03836984289227985,
|
|
"learning_rate": 5.82766439909297e-07,
|
|
"loss": 0.0043,
|
|
"num_tokens": 43633316.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 737,
|
|
"step_time": 41.86609749123454
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1312.0,
|
|
"completions/max_terminated_length": 1312.0,
|
|
"completions/mean_length": 766.21875,
|
|
"completions/mean_terminated_length": 766.21875,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 0.812423137947917,
|
|
"epoch": 0.41836734693877553,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.00836514018010348,
|
|
"learning_rate": 5.821995464852607e-07,
|
|
"loss": 0.0136,
|
|
"num_tokens": 43692154.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 738,
|
|
"step_time": 46.386055626906455
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1457.0,
|
|
"completions/max_terminated_length": 1457.0,
|
|
"completions/mean_length": 715.421875,
|
|
"completions/mean_terminated_length": 715.421875,
|
|
"completions/min_length": 24.0,
|
|
"completions/min_terminated_length": 24.0,
|
|
"entropy": 0.9142262861132622,
|
|
"epoch": 0.4189342403628118,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.008705658139660954,
|
|
"learning_rate": 5.816326530612244e-07,
|
|
"loss": -0.0175,
|
|
"num_tokens": 43751413.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 739,
|
|
"step_time": 52.30940029211342
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2451.0,
|
|
"completions/max_terminated_length": 2451.0,
|
|
"completions/mean_length": 812.6875,
|
|
"completions/mean_terminated_length": 812.6875,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 0.7261555474251509,
|
|
"epoch": 0.41950113378684806,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.008811074570985511,
|
|
"learning_rate": 5.810657596371882e-07,
|
|
"loss": 0.0078,
|
|
"num_tokens": 43813073.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5107961297035217,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 740,
|
|
"step_time": 89.20348683092743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1818.0,
|
|
"completions/max_terminated_length": 1818.0,
|
|
"completions/mean_length": 823.6875,
|
|
"completions/mean_terminated_length": 823.6875,
|
|
"completions/min_length": 423.0,
|
|
"completions/min_terminated_length": 423.0,
|
|
"entropy": 0.5654967613518238,
|
|
"epoch": 0.4200680272108844,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.007218046172056347,
|
|
"learning_rate": 5.804988662131519e-07,
|
|
"loss": 0.0341,
|
|
"num_tokens": 43876037.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 741,
|
|
"step_time": 63.95902458485216
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1419.0,
|
|
"completions/max_terminated_length": 1419.0,
|
|
"completions/mean_length": 770.546875,
|
|
"completions/mean_terminated_length": 770.546875,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.5618317853659391,
|
|
"epoch": 0.42063492063492064,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.005163202426047064,
|
|
"learning_rate": 5.799319727891157e-07,
|
|
"loss": 0.011,
|
|
"num_tokens": 43934512.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.774084210395813,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 742,
|
|
"step_time": 50.036525587551296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1354.0,
|
|
"completions/max_terminated_length": 1354.0,
|
|
"completions/mean_length": 711.6875,
|
|
"completions/mean_terminated_length": 711.6875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.9868030399084091,
|
|
"epoch": 0.4212018140589569,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.006919277046108618,
|
|
"learning_rate": 5.793650793650794e-07,
|
|
"loss": 0.0117,
|
|
"num_tokens": 43991092.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 743,
|
|
"step_time": 48.50232754461467
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1336.0,
|
|
"completions/max_terminated_length": 1336.0,
|
|
"completions/mean_length": 703.984375,
|
|
"completions/mean_terminated_length": 703.984375,
|
|
"completions/min_length": 112.0,
|
|
"completions/min_terminated_length": 112.0,
|
|
"entropy": 0.6776981428265572,
|
|
"epoch": 0.4217687074829932,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.012303693307330832,
|
|
"learning_rate": 5.78798185941043e-07,
|
|
"loss": -0.012,
|
|
"num_tokens": 44046771.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7902163863182068,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 744,
|
|
"step_time": 47.5451483046636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1411.0,
|
|
"completions/max_terminated_length": 1411.0,
|
|
"completions/mean_length": 792.9375,
|
|
"completions/mean_terminated_length": 792.9375,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.7629079688340425,
|
|
"epoch": 0.4223356009070295,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.005340939242159948,
|
|
"learning_rate": 5.782312925170067e-07,
|
|
"loss": 0.025,
|
|
"num_tokens": 44108431.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 745,
|
|
"step_time": 50.31331901997328
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1318.0,
|
|
"completions/mean_length": 875.359375,
|
|
"completions/mean_terminated_length": 824.2381591796875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.549239395186305,
|
|
"epoch": 0.42290249433106575,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.004673608949815389,
|
|
"learning_rate": 5.776643990929705e-07,
|
|
"loss": -0.0051,
|
|
"num_tokens": 44175526.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 746,
|
|
"step_time": 171.8324549011886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 797.6875,
|
|
"completions/mean_terminated_length": 797.6875,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 0.8352121766656637,
|
|
"epoch": 0.42346938775510207,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.010753196926089004,
|
|
"learning_rate": 5.770975056689342e-07,
|
|
"loss": -0.0047,
|
|
"num_tokens": 44237082.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 747,
|
|
"step_time": 48.174553577788174
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1244.0,
|
|
"completions/max_terminated_length": 1244.0,
|
|
"completions/mean_length": 730.171875,
|
|
"completions/mean_terminated_length": 730.171875,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.6675016786903143,
|
|
"epoch": 0.42403628117913833,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.006696130367345177,
|
|
"learning_rate": 5.76530612244898e-07,
|
|
"loss": 0.008,
|
|
"num_tokens": 44297397.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6423677802085876,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 748,
|
|
"step_time": 45.317835320718586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1977.0,
|
|
"completions/max_terminated_length": 1977.0,
|
|
"completions/mean_length": 780.171875,
|
|
"completions/mean_terminated_length": 780.171875,
|
|
"completions/min_length": 236.0,
|
|
"completions/min_terminated_length": 236.0,
|
|
"entropy": 0.8925861045718193,
|
|
"epoch": 0.4246031746031746,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.271484375,
|
|
"kl": 0.007037481984298211,
|
|
"learning_rate": 5.759637188208617e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 44357504.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 749,
|
|
"step_time": 70.53579318802804
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1704.0,
|
|
"completions/max_terminated_length": 1704.0,
|
|
"completions/mean_length": 742.703125,
|
|
"completions/mean_terminated_length": 742.703125,
|
|
"completions/min_length": 242.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"entropy": 0.9122004620730877,
|
|
"epoch": 0.42517006802721086,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.015559834107989445,
|
|
"learning_rate": 5.753968253968254e-07,
|
|
"loss": 0.0026,
|
|
"num_tokens": 44420829.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 750,
|
|
"step_time": 65.1315193688497
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1248.0,
|
|
"completions/max_terminated_length": 1248.0,
|
|
"completions/mean_length": 716.96875,
|
|
"completions/mean_terminated_length": 716.96875,
|
|
"completions/min_length": 293.0,
|
|
"completions/min_terminated_length": 293.0,
|
|
"entropy": 1.0605860836803913,
|
|
"epoch": 0.4257369614512472,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.008473669819068164,
|
|
"learning_rate": 5.748299319727891e-07,
|
|
"loss": -0.0201,
|
|
"num_tokens": 44477803.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 751,
|
|
"step_time": 44.40520949661732
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1243.0,
|
|
"completions/max_terminated_length": 1243.0,
|
|
"completions/mean_length": 791.4375,
|
|
"completions/mean_terminated_length": 791.4375,
|
|
"completions/min_length": 406.0,
|
|
"completions/min_terminated_length": 406.0,
|
|
"entropy": 0.8621129430830479,
|
|
"epoch": 0.42630385487528344,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.006231349092558958,
|
|
"learning_rate": 5.742630385487527e-07,
|
|
"loss": 0.047,
|
|
"num_tokens": 44539903.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 752,
|
|
"step_time": 44.989488820545375
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1416.0,
|
|
"completions/max_terminated_length": 1416.0,
|
|
"completions/mean_length": 691.96875,
|
|
"completions/mean_terminated_length": 691.96875,
|
|
"completions/min_length": 331.0,
|
|
"completions/min_terminated_length": 331.0,
|
|
"entropy": 0.7993203308433294,
|
|
"epoch": 0.4268707482993197,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.43359375,
|
|
"kl": 0.01025540748378262,
|
|
"learning_rate": 5.736961451247165e-07,
|
|
"loss": 0.0044,
|
|
"num_tokens": 44595045.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 753,
|
|
"step_time": 50.426668647676706
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1345.0,
|
|
"completions/max_terminated_length": 1345.0,
|
|
"completions/mean_length": 753.328125,
|
|
"completions/mean_terminated_length": 753.328125,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 0.7891684416681528,
|
|
"epoch": 0.427437641723356,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.359375,
|
|
"kl": 0.006862209120299667,
|
|
"learning_rate": 5.731292517006802e-07,
|
|
"loss": 0.0104,
|
|
"num_tokens": 44653474.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 754,
|
|
"step_time": 47.68941889423877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1691.0,
|
|
"completions/max_terminated_length": 1691.0,
|
|
"completions/mean_length": 769.1875,
|
|
"completions/mean_terminated_length": 769.1875,
|
|
"completions/min_length": 335.0,
|
|
"completions/min_terminated_length": 335.0,
|
|
"entropy": 0.6205135956406593,
|
|
"epoch": 0.4280045351473923,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.006889806914841756,
|
|
"learning_rate": 5.72562358276644e-07,
|
|
"loss": 0.0459,
|
|
"num_tokens": 44711806.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 755,
|
|
"step_time": 59.8132201526314
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1309.0,
|
|
"completions/max_terminated_length": 1309.0,
|
|
"completions/mean_length": 642.921875,
|
|
"completions/mean_terminated_length": 642.921875,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.8332008868455887,
|
|
"epoch": 0.42857142857142855,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.007948527971166186,
|
|
"learning_rate": 5.719954648526076e-07,
|
|
"loss": -0.0079,
|
|
"num_tokens": 44761905.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 756,
|
|
"step_time": 46.39207501336932
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1474.0,
|
|
"completions/max_terminated_length": 1474.0,
|
|
"completions/mean_length": 687.5625,
|
|
"completions/mean_terminated_length": 687.5625,
|
|
"completions/min_length": 198.0,
|
|
"completions/min_terminated_length": 198.0,
|
|
"entropy": 1.0087007656693459,
|
|
"epoch": 0.42913832199546487,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.007352959044510499,
|
|
"learning_rate": 5.714285714285714e-07,
|
|
"loss": -0.0237,
|
|
"num_tokens": 44816085.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6029338836669922,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 757,
|
|
"step_time": 51.99128063302487
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1062.0,
|
|
"completions/max_terminated_length": 1062.0,
|
|
"completions/mean_length": 685.40625,
|
|
"completions/mean_terminated_length": 685.40625,
|
|
"completions/min_length": 51.0,
|
|
"completions/min_terminated_length": 51.0,
|
|
"entropy": 0.7438847832381725,
|
|
"epoch": 0.42970521541950113,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.007625193960848264,
|
|
"learning_rate": 5.708616780045351e-07,
|
|
"loss": 0.0178,
|
|
"num_tokens": 44868951.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 758,
|
|
"step_time": 37.652617966756225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1511.0,
|
|
"completions/max_terminated_length": 1511.0,
|
|
"completions/mean_length": 883.546875,
|
|
"completions/mean_terminated_length": 883.546875,
|
|
"completions/min_length": 418.0,
|
|
"completions/min_terminated_length": 418.0,
|
|
"entropy": 0.7068359889090061,
|
|
"epoch": 0.4302721088435374,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.006396272510755807,
|
|
"learning_rate": 5.702947845804989e-07,
|
|
"loss": 0.0084,
|
|
"num_tokens": 44936370.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 759,
|
|
"step_time": 54.0413280595094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1516.0,
|
|
"completions/max_terminated_length": 1516.0,
|
|
"completions/mean_length": 763.28125,
|
|
"completions/mean_terminated_length": 763.28125,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 0.8476640395820141,
|
|
"epoch": 0.4308390022675737,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40625,
|
|
"kl": 0.006580744084203616,
|
|
"learning_rate": 5.697278911564625e-07,
|
|
"loss": 0.0411,
|
|
"num_tokens": 44996036.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 760,
|
|
"step_time": 53.84218557458371
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1287.0,
|
|
"completions/max_terminated_length": 1287.0,
|
|
"completions/mean_length": 798.828125,
|
|
"completions/mean_terminated_length": 798.828125,
|
|
"completions/min_length": 218.0,
|
|
"completions/min_terminated_length": 218.0,
|
|
"entropy": 0.7278933059424162,
|
|
"epoch": 0.43140589569161,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.01245811271655839,
|
|
"learning_rate": 5.691609977324263e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 45061577.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4753340482711792,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 761,
|
|
"step_time": 47.47228251863271
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2392.0,
|
|
"completions/max_terminated_length": 2392.0,
|
|
"completions/mean_length": 875.53125,
|
|
"completions/mean_terminated_length": 875.53125,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.68974794074893,
|
|
"epoch": 0.43197278911564624,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.011912069719983265,
|
|
"learning_rate": 5.685941043083899e-07,
|
|
"loss": -0.0019,
|
|
"num_tokens": 45127203.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6878606677055359,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 762,
|
|
"step_time": 87.16267612203956
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1349.0,
|
|
"completions/max_terminated_length": 1349.0,
|
|
"completions/mean_length": 737.34375,
|
|
"completions/mean_terminated_length": 737.34375,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.9138299617916346,
|
|
"epoch": 0.43253968253968256,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.384765625,
|
|
"kl": 0.009958441682101693,
|
|
"learning_rate": 5.680272108843537e-07,
|
|
"loss": -0.0085,
|
|
"num_tokens": 45185873.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4787135720252991,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 763,
|
|
"step_time": 48.26601691264659
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1403.0,
|
|
"completions/max_terminated_length": 1403.0,
|
|
"completions/mean_length": 743.203125,
|
|
"completions/mean_terminated_length": 743.203125,
|
|
"completions/min_length": 195.0,
|
|
"completions/min_terminated_length": 195.0,
|
|
"entropy": 0.7706839740276337,
|
|
"epoch": 0.4331065759637188,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.007704952251515351,
|
|
"learning_rate": 5.674603174603174e-07,
|
|
"loss": -0.0072,
|
|
"num_tokens": 45243630.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 764,
|
|
"step_time": 49.51623547915369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1593.0,
|
|
"completions/max_terminated_length": 1593.0,
|
|
"completions/mean_length": 805.71875,
|
|
"completions/mean_terminated_length": 805.71875,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.8907484859228134,
|
|
"epoch": 0.4336734693877551,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.008698665333213285,
|
|
"learning_rate": 5.668934240362812e-07,
|
|
"loss": 0.0189,
|
|
"num_tokens": 45307252.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 765,
|
|
"step_time": 57.227187460288405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1051.0,
|
|
"completions/max_terminated_length": 1051.0,
|
|
"completions/mean_length": 602.234375,
|
|
"completions/mean_terminated_length": 602.234375,
|
|
"completions/min_length": 108.0,
|
|
"completions/min_terminated_length": 108.0,
|
|
"entropy": 1.1054105497896671,
|
|
"epoch": 0.4342403628117914,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.66015625,
|
|
"kl": 0.01244768631295301,
|
|
"learning_rate": 5.663265306122449e-07,
|
|
"loss": 0.0158,
|
|
"num_tokens": 45357795.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 766,
|
|
"step_time": 37.28772302530706
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1020.0,
|
|
"completions/max_terminated_length": 1020.0,
|
|
"completions/mean_length": 625.1875,
|
|
"completions/mean_terminated_length": 625.1875,
|
|
"completions/min_length": 350.0,
|
|
"completions/min_terminated_length": 350.0,
|
|
"entropy": 0.8914185203611851,
|
|
"epoch": 0.43480725623582767,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.73046875,
|
|
"kl": 0.01470986133790575,
|
|
"learning_rate": 5.657596371882087e-07,
|
|
"loss": 0.024,
|
|
"num_tokens": 45407479.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7531349658966064,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 767,
|
|
"step_time": 36.99625062011182
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1566.0,
|
|
"completions/max_terminated_length": 1566.0,
|
|
"completions/mean_length": 699.46875,
|
|
"completions/mean_terminated_length": 699.46875,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 0.7746062316000462,
|
|
"epoch": 0.43537414965986393,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.013445902935927734,
|
|
"learning_rate": 5.651927437641723e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 45462445.0,
|
|
"reward": 0.484375,
|
|
"reward_std": 0.8772646188735962,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 768,
|
|
"step_time": 54.5709900315851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1390.0,
|
|
"completions/mean_length": 976.171875,
|
|
"completions/mean_terminated_length": 875.5322265625,
|
|
"completions/min_length": 63.0,
|
|
"completions/min_terminated_length": 63.0,
|
|
"entropy": 0.5542530845850706,
|
|
"epoch": 0.43594104308390025,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.006793142434617039,
|
|
"learning_rate": 5.64625850340136e-07,
|
|
"loss": 0.0404,
|
|
"num_tokens": 45533792.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 769,
|
|
"step_time": 170.40638070274144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1597.0,
|
|
"completions/max_terminated_length": 1597.0,
|
|
"completions/mean_length": 740.296875,
|
|
"completions/mean_terminated_length": 740.296875,
|
|
"completions/min_length": 246.0,
|
|
"completions/min_terminated_length": 246.0,
|
|
"entropy": 0.5962138585746288,
|
|
"epoch": 0.4365079365079365,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.01063502197212074,
|
|
"learning_rate": 5.640589569160997e-07,
|
|
"loss": 0.032,
|
|
"num_tokens": 45591795.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6884463429450989,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 770,
|
|
"step_time": 56.46308619529009
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1457.0,
|
|
"completions/max_terminated_length": 1457.0,
|
|
"completions/mean_length": 728.296875,
|
|
"completions/mean_terminated_length": 728.296875,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.9644140545278788,
|
|
"epoch": 0.4370748299319728,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.013081034616334364,
|
|
"learning_rate": 5.634920634920635e-07,
|
|
"loss": 0.0059,
|
|
"num_tokens": 45647358.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4787135720252991,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 771,
|
|
"step_time": 51.11765184998512
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 705.78125,
|
|
"completions/mean_terminated_length": 705.78125,
|
|
"completions/min_length": 240.0,
|
|
"completions/min_terminated_length": 240.0,
|
|
"entropy": 0.6892326436936855,
|
|
"epoch": 0.4376417233560091,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"kl": 0.005590775981545448,
|
|
"learning_rate": 5.629251700680272e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 45701464.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 772,
|
|
"step_time": 48.400738924741745
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1268.0,
|
|
"completions/max_terminated_length": 1268.0,
|
|
"completions/mean_length": 739.453125,
|
|
"completions/mean_terminated_length": 739.453125,
|
|
"completions/min_length": 300.0,
|
|
"completions/min_terminated_length": 300.0,
|
|
"entropy": 0.9138533473014832,
|
|
"epoch": 0.43820861678004536,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.314453125,
|
|
"kl": 0.009496628234046511,
|
|
"learning_rate": 5.62358276643991e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 45759157.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 773,
|
|
"step_time": 44.88098752871156
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1583.0,
|
|
"completions/max_terminated_length": 1583.0,
|
|
"completions/mean_length": 810.5625,
|
|
"completions/mean_terminated_length": 810.5625,
|
|
"completions/min_length": 399.0,
|
|
"completions/min_terminated_length": 399.0,
|
|
"entropy": 0.840919841080904,
|
|
"epoch": 0.4387755102040816,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.005558051401749253,
|
|
"learning_rate": 5.617913832199547e-07,
|
|
"loss": 0.0231,
|
|
"num_tokens": 45823593.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.40999117493629456,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 774,
|
|
"step_time": 57.04373292531818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1369.0,
|
|
"completions/mean_length": 885.015625,
|
|
"completions/mean_terminated_length": 834.0476684570312,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.6667205300182104,
|
|
"epoch": 0.43934240362811794,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.408203125,
|
|
"kl": 0.00555698896641843,
|
|
"learning_rate": 5.612244897959183e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 45892058.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 775,
|
|
"step_time": 176.05875206179917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1728.0,
|
|
"completions/max_terminated_length": 1728.0,
|
|
"completions/mean_length": 833.765625,
|
|
"completions/mean_terminated_length": 833.765625,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.6955114342272282,
|
|
"epoch": 0.4399092970521542,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3984375,
|
|
"kl": 0.008808961953036487,
|
|
"learning_rate": 5.60657596371882e-07,
|
|
"loss": -0.0076,
|
|
"num_tokens": 45958427.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6029338836669922,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 776,
|
|
"step_time": 62.196417150087655
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1166.0,
|
|
"completions/max_terminated_length": 1166.0,
|
|
"completions/mean_length": 707.171875,
|
|
"completions/mean_terminated_length": 707.171875,
|
|
"completions/min_length": 231.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"entropy": 0.9409902021288872,
|
|
"epoch": 0.44047619047619047,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.021475753892445937,
|
|
"learning_rate": 5.600907029478457e-07,
|
|
"loss": -0.018,
|
|
"num_tokens": 46014638.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 777,
|
|
"step_time": 41.509524879045784
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1257.0,
|
|
"completions/max_terminated_length": 1257.0,
|
|
"completions/mean_length": 714.640625,
|
|
"completions/mean_terminated_length": 714.640625,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.6931835245341063,
|
|
"epoch": 0.4410430839002268,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66796875,
|
|
"kl": 0.011560758081031963,
|
|
"learning_rate": 5.595238095238095e-07,
|
|
"loss": 0.0188,
|
|
"num_tokens": 46070143.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 778,
|
|
"step_time": 44.50387246534228
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1644.0,
|
|
"completions/max_terminated_length": 1644.0,
|
|
"completions/mean_length": 766.65625,
|
|
"completions/mean_terminated_length": 766.65625,
|
|
"completions/min_length": 169.0,
|
|
"completions/min_terminated_length": 169.0,
|
|
"entropy": 0.735244506970048,
|
|
"epoch": 0.44160997732426305,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.01051445792109007,
|
|
"learning_rate": 5.589569160997732e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 46127697.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 779,
|
|
"step_time": 57.430952980183065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1552.0,
|
|
"completions/max_terminated_length": 1552.0,
|
|
"completions/mean_length": 711.875,
|
|
"completions/mean_terminated_length": 711.875,
|
|
"completions/min_length": 176.0,
|
|
"completions/min_terminated_length": 176.0,
|
|
"entropy": 0.8216298818588257,
|
|
"epoch": 0.4421768707482993,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.011475614286609925,
|
|
"learning_rate": 5.58390022675737e-07,
|
|
"loss": -0.0211,
|
|
"num_tokens": 46183745.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 780,
|
|
"step_time": 54.353818798437715
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1312.0,
|
|
"completions/max_terminated_length": 1312.0,
|
|
"completions/mean_length": 778.921875,
|
|
"completions/mean_terminated_length": 778.921875,
|
|
"completions/min_length": 437.0,
|
|
"completions/min_terminated_length": 437.0,
|
|
"entropy": 0.7938818261027336,
|
|
"epoch": 0.4427437641723356,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58984375,
|
|
"kl": 0.008235653716837987,
|
|
"learning_rate": 5.578231292517006e-07,
|
|
"loss": 0.0049,
|
|
"num_tokens": 46242604.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 781,
|
|
"step_time": 45.884823876433074
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1696.0,
|
|
"completions/max_terminated_length": 1696.0,
|
|
"completions/mean_length": 822.296875,
|
|
"completions/mean_terminated_length": 822.296875,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.8532251175493002,
|
|
"epoch": 0.4433106575963719,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.005970577418338507,
|
|
"learning_rate": 5.572562358276644e-07,
|
|
"loss": 0.0023,
|
|
"num_tokens": 46305655.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 782,
|
|
"step_time": 59.79147854913026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1313.0,
|
|
"completions/max_terminated_length": 1313.0,
|
|
"completions/mean_length": 663.5,
|
|
"completions/mean_terminated_length": 663.5,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 1.0758831836283207,
|
|
"epoch": 0.44387755102040816,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.007333965826546773,
|
|
"learning_rate": 5.566893424036281e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 46357047.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 783,
|
|
"step_time": 46.13503709062934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1106.0,
|
|
"completions/max_terminated_length": 1106.0,
|
|
"completions/mean_length": 592.890625,
|
|
"completions/mean_terminated_length": 592.890625,
|
|
"completions/min_length": 211.0,
|
|
"completions/min_terminated_length": 211.0,
|
|
"entropy": 0.8459891099482775,
|
|
"epoch": 0.4444444444444444,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6875,
|
|
"kl": 0.01103259137016721,
|
|
"learning_rate": 5.561224489795918e-07,
|
|
"loss": -0.0038,
|
|
"num_tokens": 46405528.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 784,
|
|
"step_time": 39.219961393624544
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1226.0,
|
|
"completions/max_terminated_length": 1226.0,
|
|
"completions/mean_length": 711.078125,
|
|
"completions/mean_terminated_length": 711.078125,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.8587812408804893,
|
|
"epoch": 0.44501133786848074,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.61328125,
|
|
"kl": 0.007803906075423583,
|
|
"learning_rate": 5.555555555555555e-07,
|
|
"loss": 0.0025,
|
|
"num_tokens": 46460373.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 785,
|
|
"step_time": 43.14912079460919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1212.0,
|
|
"completions/max_terminated_length": 1212.0,
|
|
"completions/mean_length": 707.984375,
|
|
"completions/mean_terminated_length": 707.984375,
|
|
"completions/min_length": 252.0,
|
|
"completions/min_terminated_length": 252.0,
|
|
"entropy": 0.8257488086819649,
|
|
"epoch": 0.445578231292517,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"kl": 0.010672343749320135,
|
|
"learning_rate": 5.549886621315193e-07,
|
|
"loss": 0.0097,
|
|
"num_tokens": 46519988.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 786,
|
|
"step_time": 45.32893858477473
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1256.0,
|
|
"completions/max_terminated_length": 1256.0,
|
|
"completions/mean_length": 624.0625,
|
|
"completions/mean_terminated_length": 624.0625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.9342876747250557,
|
|
"epoch": 0.44614512471655327,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.012740399164613336,
|
|
"learning_rate": 5.544217687074829e-07,
|
|
"loss": -0.0184,
|
|
"num_tokens": 46569904.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 787,
|
|
"step_time": 44.07589914929122
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 785.9375,
|
|
"completions/mean_terminated_length": 785.9375,
|
|
"completions/min_length": 347.0,
|
|
"completions/min_terminated_length": 347.0,
|
|
"entropy": 0.6767003983259201,
|
|
"epoch": 0.4467120181405896,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.0065141103841597214,
|
|
"learning_rate": 5.538548752834467e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 46630532.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 788,
|
|
"step_time": 44.48235871270299
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1319.0,
|
|
"completions/max_terminated_length": 1319.0,
|
|
"completions/mean_length": 801.65625,
|
|
"completions/mean_terminated_length": 801.65625,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.6320349834859371,
|
|
"epoch": 0.44727891156462585,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.005608612904325128,
|
|
"learning_rate": 5.532879818594104e-07,
|
|
"loss": -0.0187,
|
|
"num_tokens": 46691014.0,
|
|
"reward": 0.4140625,
|
|
"reward_std": 0.7995766997337341,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 789,
|
|
"step_time": 46.12413862347603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2356.0,
|
|
"completions/max_terminated_length": 2356.0,
|
|
"completions/mean_length": 703.5,
|
|
"completions/mean_terminated_length": 703.5,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 0.7034353483468294,
|
|
"epoch": 0.4478458049886621,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.63671875,
|
|
"kl": 0.008118132798699662,
|
|
"learning_rate": 5.527210884353742e-07,
|
|
"loss": -0.0222,
|
|
"num_tokens": 46748486.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 790,
|
|
"step_time": 87.03440655209124
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1929.0,
|
|
"completions/max_terminated_length": 1929.0,
|
|
"completions/mean_length": 739.34375,
|
|
"completions/mean_terminated_length": 739.34375,
|
|
"completions/min_length": 242.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"entropy": 0.693330405279994,
|
|
"epoch": 0.44841269841269843,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"kl": 0.005587221021414734,
|
|
"learning_rate": 5.521541950113379e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 46806652.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 791,
|
|
"step_time": 70.8655961798504
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1976.0,
|
|
"completions/max_terminated_length": 1976.0,
|
|
"completions/mean_length": 891.0625,
|
|
"completions/mean_terminated_length": 891.0625,
|
|
"completions/min_length": 405.0,
|
|
"completions/min_terminated_length": 405.0,
|
|
"entropy": 0.6741422917693853,
|
|
"epoch": 0.4489795918367347,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.007003923776210286,
|
|
"learning_rate": 5.515873015873016e-07,
|
|
"loss": -0.012,
|
|
"num_tokens": 46873936.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.8158508539199829,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 792,
|
|
"step_time": 70.97057412751019
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1271.0,
|
|
"completions/max_terminated_length": 1271.0,
|
|
"completions/mean_length": 745.375,
|
|
"completions/mean_terminated_length": 745.375,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 0.8206442259252071,
|
|
"epoch": 0.44954648526077096,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.006794253451516852,
|
|
"learning_rate": 5.510204081632652e-07,
|
|
"loss": 0.0157,
|
|
"num_tokens": 46932504.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 793,
|
|
"step_time": 45.19920152518898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2271.0,
|
|
"completions/max_terminated_length": 2271.0,
|
|
"completions/mean_length": 844.25,
|
|
"completions/mean_terminated_length": 844.25,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.6960771568119526,
|
|
"epoch": 0.4501133786848073,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.322265625,
|
|
"kl": 0.011700832488713786,
|
|
"learning_rate": 5.504535147392289e-07,
|
|
"loss": -0.0031,
|
|
"num_tokens": 47001184.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.7126966714859009,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 794,
|
|
"step_time": 84.1520056463778
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1249.0,
|
|
"completions/max_terminated_length": 1249.0,
|
|
"completions/mean_length": 718.1875,
|
|
"completions/mean_terminated_length": 718.1875,
|
|
"completions/min_length": 439.0,
|
|
"completions/min_terminated_length": 439.0,
|
|
"entropy": 0.971361555159092,
|
|
"epoch": 0.45068027210884354,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33203125,
|
|
"kl": 0.008962832813267596,
|
|
"learning_rate": 5.498866213151927e-07,
|
|
"loss": 0.0192,
|
|
"num_tokens": 47058076.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 795,
|
|
"step_time": 44.5225541787222
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1780.0,
|
|
"completions/max_terminated_length": 1780.0,
|
|
"completions/mean_length": 813.984375,
|
|
"completions/mean_terminated_length": 813.984375,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.8689982630312443,
|
|
"epoch": 0.4512471655328798,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.007326565100811422,
|
|
"learning_rate": 5.493197278911564e-07,
|
|
"loss": 0.0189,
|
|
"num_tokens": 47127635.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 796,
|
|
"step_time": 69.09512663632631
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1673.0,
|
|
"completions/max_terminated_length": 1673.0,
|
|
"completions/mean_length": 856.265625,
|
|
"completions/mean_terminated_length": 856.265625,
|
|
"completions/min_length": 408.0,
|
|
"completions/min_terminated_length": 408.0,
|
|
"entropy": 0.7650571893900633,
|
|
"epoch": 0.4518140589569161,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.014412336822715588,
|
|
"learning_rate": 5.487528344671202e-07,
|
|
"loss": 0.0218,
|
|
"num_tokens": 47192276.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.8100543022155762,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 797,
|
|
"step_time": 59.76943079568446
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1552.0,
|
|
"completions/max_terminated_length": 1552.0,
|
|
"completions/mean_length": 709.703125,
|
|
"completions/mean_terminated_length": 709.703125,
|
|
"completions/min_length": 252.0,
|
|
"completions/min_terminated_length": 252.0,
|
|
"entropy": 0.6407954432070255,
|
|
"epoch": 0.4523809523809524,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"kl": 0.012908270364277996,
|
|
"learning_rate": 5.481859410430839e-07,
|
|
"loss": 0.0145,
|
|
"num_tokens": 47248073.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 798,
|
|
"step_time": 54.691888799890876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1284.0,
|
|
"completions/max_terminated_length": 1284.0,
|
|
"completions/mean_length": 753.65625,
|
|
"completions/mean_terminated_length": 753.65625,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 0.6514208652079105,
|
|
"epoch": 0.45294784580498865,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.005030949279898778,
|
|
"learning_rate": 5.476190476190477e-07,
|
|
"loss": -0.0055,
|
|
"num_tokens": 47306323.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5579067468643188,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 799,
|
|
"step_time": 45.29969593696296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1396.0,
|
|
"completions/max_terminated_length": 1396.0,
|
|
"completions/mean_length": 746.34375,
|
|
"completions/mean_terminated_length": 746.34375,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.9061711113899946,
|
|
"epoch": 0.45351473922902497,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.00890976784285158,
|
|
"learning_rate": 5.470521541950112e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 47364089.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 800,
|
|
"step_time": 49.48979236185551
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1556.0,
|
|
"completions/max_terminated_length": 1556.0,
|
|
"completions/mean_length": 759.6875,
|
|
"completions/mean_terminated_length": 759.6875,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 0.7575592007488012,
|
|
"epoch": 0.45408163265306123,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.014216447481885552,
|
|
"learning_rate": 5.46485260770975e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 47424749.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7478474378585815,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 801,
|
|
"step_time": 55.553089062683284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1827.0,
|
|
"completions/max_terminated_length": 1827.0,
|
|
"completions/mean_length": 735.78125,
|
|
"completions/mean_terminated_length": 735.78125,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.7598094884306192,
|
|
"epoch": 0.4546485260770975,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.005780653285910375,
|
|
"learning_rate": 5.459183673469387e-07,
|
|
"loss": 0.0515,
|
|
"num_tokens": 47481783.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 802,
|
|
"step_time": 65.4749649092555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1434.0,
|
|
"completions/max_terminated_length": 1434.0,
|
|
"completions/mean_length": 773.0,
|
|
"completions/mean_terminated_length": 773.0,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.79466787353158,
|
|
"epoch": 0.4552154195011338,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"kl": 0.009720435627968982,
|
|
"learning_rate": 5.453514739229025e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 47543551.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 803,
|
|
"step_time": 51.89830320235342
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1351.0,
|
|
"completions/max_terminated_length": 1351.0,
|
|
"completions/mean_length": 694.1875,
|
|
"completions/mean_terminated_length": 694.1875,
|
|
"completions/min_length": 417.0,
|
|
"completions/min_terminated_length": 417.0,
|
|
"entropy": 0.8484663590788841,
|
|
"epoch": 0.4557823129251701,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.010874871266423725,
|
|
"learning_rate": 5.447845804988662e-07,
|
|
"loss": -0.029,
|
|
"num_tokens": 47596947.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.125,
|
|
"rewards/format_reward_func/std": 0.2182178944349289,
|
|
"step": 804,
|
|
"step_time": 47.52819735277444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1360.0,
|
|
"completions/max_terminated_length": 1360.0,
|
|
"completions/mean_length": 745.640625,
|
|
"completions/mean_terminated_length": 745.640625,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.6674797665327787,
|
|
"epoch": 0.45634920634920634,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.010321498702978715,
|
|
"learning_rate": 5.4421768707483e-07,
|
|
"loss": 0.0135,
|
|
"num_tokens": 47655684.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.417855441570282,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 805,
|
|
"step_time": 48.383637744002044
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1852.0,
|
|
"completions/max_terminated_length": 1852.0,
|
|
"completions/mean_length": 767.984375,
|
|
"completions/mean_terminated_length": 767.984375,
|
|
"completions/min_length": 256.0,
|
|
"completions/min_terminated_length": 256.0,
|
|
"entropy": 0.8424343373626471,
|
|
"epoch": 0.45691609977324266,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.0059676786622731015,
|
|
"learning_rate": 5.436507936507936e-07,
|
|
"loss": -0.0049,
|
|
"num_tokens": 47717147.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.666620135307312,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 806,
|
|
"step_time": 66.206379073672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1258.0,
|
|
"completions/max_terminated_length": 1258.0,
|
|
"completions/mean_length": 688.96875,
|
|
"completions/mean_terminated_length": 688.96875,
|
|
"completions/min_length": 209.0,
|
|
"completions/min_terminated_length": 209.0,
|
|
"entropy": 0.9894794672727585,
|
|
"epoch": 0.4574829931972789,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.010550793827860616,
|
|
"learning_rate": 5.430839002267574e-07,
|
|
"loss": -0.0152,
|
|
"num_tokens": 47771961.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 807,
|
|
"step_time": 43.96737048588693
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 770.28125,
|
|
"completions/mean_terminated_length": 770.28125,
|
|
"completions/min_length": 333.0,
|
|
"completions/min_terminated_length": 333.0,
|
|
"entropy": 0.7077070437371731,
|
|
"epoch": 0.4580498866213152,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.016043149938923307,
|
|
"learning_rate": 5.42517006802721e-07,
|
|
"loss": -0.0216,
|
|
"num_tokens": 47832499.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7194505929946899,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 808,
|
|
"step_time": 49.31481224950403
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1654.0,
|
|
"completions/mean_length": 725.5625,
|
|
"completions/mean_terminated_length": 672.0635375976562,
|
|
"completions/min_length": 217.0,
|
|
"completions/min_terminated_length": 217.0,
|
|
"entropy": 0.700819993391633,
|
|
"epoch": 0.45861678004535145,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.015738855450763367,
|
|
"learning_rate": 5.419501133786848e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 47891295.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 809,
|
|
"step_time": 172.91593146976084
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1244.0,
|
|
"completions/max_terminated_length": 1244.0,
|
|
"completions/mean_length": 745.28125,
|
|
"completions/mean_terminated_length": 745.28125,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.6663068141788244,
|
|
"epoch": 0.45918367346938777,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.012314054460148327,
|
|
"learning_rate": 5.413832199546485e-07,
|
|
"loss": -0.0195,
|
|
"num_tokens": 47948993.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6164092421531677,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 810,
|
|
"step_time": 45.93409332446754
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2310.0,
|
|
"completions/max_terminated_length": 2310.0,
|
|
"completions/mean_length": 897.8125,
|
|
"completions/mean_terminated_length": 897.8125,
|
|
"completions/min_length": 246.0,
|
|
"completions/min_terminated_length": 246.0,
|
|
"entropy": 0.5644901534542441,
|
|
"epoch": 0.45975056689342403,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.3984375,
|
|
"kl": 0.008329780474014115,
|
|
"learning_rate": 5.408163265306123e-07,
|
|
"loss": -0.0265,
|
|
"num_tokens": 48016541.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 811,
|
|
"step_time": 83.55836615059525
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1334.0,
|
|
"completions/max_terminated_length": 1334.0,
|
|
"completions/mean_length": 738.34375,
|
|
"completions/mean_terminated_length": 738.34375,
|
|
"completions/min_length": 325.0,
|
|
"completions/min_terminated_length": 325.0,
|
|
"entropy": 0.891378290951252,
|
|
"epoch": 0.4603174603174603,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.007234719378175214,
|
|
"learning_rate": 5.402494331065759e-07,
|
|
"loss": 0.0216,
|
|
"num_tokens": 48076019.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6172134280204773,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 812,
|
|
"step_time": 48.76369559764862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1200.0,
|
|
"completions/max_terminated_length": 1200.0,
|
|
"completions/mean_length": 730.875,
|
|
"completions/mean_terminated_length": 730.875,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.7049499042332172,
|
|
"epoch": 0.4608843537414966,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.008877526095602661,
|
|
"learning_rate": 5.396825396825396e-07,
|
|
"loss": -0.018,
|
|
"num_tokens": 48133363.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 813,
|
|
"step_time": 42.50087522342801
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1783.0,
|
|
"completions/max_terminated_length": 1783.0,
|
|
"completions/mean_length": 889.875,
|
|
"completions/mean_terminated_length": 889.875,
|
|
"completions/min_length": 462.0,
|
|
"completions/min_terminated_length": 462.0,
|
|
"entropy": 0.8389473538845778,
|
|
"epoch": 0.4614512471655329,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.005758157116360962,
|
|
"learning_rate": 5.391156462585034e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 48201595.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 814,
|
|
"step_time": 63.47205546218902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1623.0,
|
|
"completions/max_terminated_length": 1623.0,
|
|
"completions/mean_length": 736.546875,
|
|
"completions/mean_terminated_length": 736.546875,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.8449075985699892,
|
|
"epoch": 0.46201814058956914,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.008303162961965427,
|
|
"learning_rate": 5.385487528344671e-07,
|
|
"loss": 0.0191,
|
|
"num_tokens": 48258270.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.8902789950370789,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 815,
|
|
"step_time": 57.7476409683004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1195.0,
|
|
"completions/max_terminated_length": 1195.0,
|
|
"completions/mean_length": 705.515625,
|
|
"completions/mean_terminated_length": 705.515625,
|
|
"completions/min_length": 352.0,
|
|
"completions/min_terminated_length": 352.0,
|
|
"entropy": 0.8178889341652393,
|
|
"epoch": 0.46258503401360546,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.0060025438433513045,
|
|
"learning_rate": 5.379818594104308e-07,
|
|
"loss": 0.0285,
|
|
"num_tokens": 48317047.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 816,
|
|
"step_time": 44.6344731207937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1553.0,
|
|
"completions/max_terminated_length": 1553.0,
|
|
"completions/mean_length": 854.84375,
|
|
"completions/mean_terminated_length": 854.84375,
|
|
"completions/min_length": 514.0,
|
|
"completions/min_terminated_length": 514.0,
|
|
"entropy": 0.7517775446176529,
|
|
"epoch": 0.4631519274376417,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.008044944843277335,
|
|
"learning_rate": 5.374149659863945e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 48382277.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 817,
|
|
"step_time": 55.41614804044366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1560.0,
|
|
"completions/max_terminated_length": 1560.0,
|
|
"completions/mean_length": 770.359375,
|
|
"completions/mean_terminated_length": 770.359375,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.8113478384912014,
|
|
"epoch": 0.463718820861678,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.008554863190511242,
|
|
"learning_rate": 5.368480725623582e-07,
|
|
"loss": 0.0163,
|
|
"num_tokens": 48443116.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7099072337150574,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 818,
|
|
"step_time": 56.03627952095121
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1332.0,
|
|
"completions/max_terminated_length": 1332.0,
|
|
"completions/mean_length": 744.328125,
|
|
"completions/mean_terminated_length": 744.328125,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.7487884275615215,
|
|
"epoch": 0.4642857142857143,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.008048018557019532,
|
|
"learning_rate": 5.362811791383219e-07,
|
|
"loss": -0.0179,
|
|
"num_tokens": 48501481.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.5076302886009216,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 819,
|
|
"step_time": 47.85775559209287
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1445.0,
|
|
"completions/max_terminated_length": 1445.0,
|
|
"completions/mean_length": 713.984375,
|
|
"completions/mean_terminated_length": 713.984375,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.9435363914817572,
|
|
"epoch": 0.46485260770975056,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.014161181636154652,
|
|
"learning_rate": 5.357142857142857e-07,
|
|
"loss": 0.0011,
|
|
"num_tokens": 48556800.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6900206208229065,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 820,
|
|
"step_time": 51.19664356391877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1615.0,
|
|
"completions/max_terminated_length": 1615.0,
|
|
"completions/mean_length": 770.234375,
|
|
"completions/mean_terminated_length": 770.234375,
|
|
"completions/min_length": 391.0,
|
|
"completions/min_terminated_length": 391.0,
|
|
"entropy": 0.7329765241593122,
|
|
"epoch": 0.46541950113378683,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.008304058515932411,
|
|
"learning_rate": 5.351473922902494e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 48616655.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 821,
|
|
"step_time": 57.367568478919566
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2515.0,
|
|
"completions/max_terminated_length": 2515.0,
|
|
"completions/mean_length": 792.984375,
|
|
"completions/mean_terminated_length": 792.984375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.5834014657884836,
|
|
"epoch": 0.46598639455782315,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.369140625,
|
|
"kl": 0.007632716064108536,
|
|
"learning_rate": 5.345804988662132e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 48676334.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 822,
|
|
"step_time": 91.93235441762954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1499.0,
|
|
"completions/max_terminated_length": 1499.0,
|
|
"completions/mean_length": 787.78125,
|
|
"completions/mean_terminated_length": 787.78125,
|
|
"completions/min_length": 185.0,
|
|
"completions/min_terminated_length": 185.0,
|
|
"entropy": 0.7342123314738274,
|
|
"epoch": 0.4665532879818594,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.357421875,
|
|
"kl": 0.006202431570272893,
|
|
"learning_rate": 5.340136054421769e-07,
|
|
"loss": -0.005,
|
|
"num_tokens": 48735784.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 823,
|
|
"step_time": 52.9507035119459
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1168.0,
|
|
"completions/max_terminated_length": 1168.0,
|
|
"completions/mean_length": 664.109375,
|
|
"completions/mean_terminated_length": 664.109375,
|
|
"completions/min_length": 204.0,
|
|
"completions/min_terminated_length": 204.0,
|
|
"entropy": 0.738832512870431,
|
|
"epoch": 0.4671201814058957,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.62890625,
|
|
"kl": 0.008733190479688346,
|
|
"learning_rate": 5.334467120181405e-07,
|
|
"loss": -0.0005,
|
|
"num_tokens": 48788079.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 824,
|
|
"step_time": 41.29877850599587
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1204.0,
|
|
"completions/max_terminated_length": 1204.0,
|
|
"completions/mean_length": 686.0625,
|
|
"completions/mean_terminated_length": 686.0625,
|
|
"completions/min_length": 259.0,
|
|
"completions/min_terminated_length": 259.0,
|
|
"entropy": 0.9853662140667439,
|
|
"epoch": 0.467687074829932,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.63671875,
|
|
"kl": 0.009879105957224965,
|
|
"learning_rate": 5.328798185941042e-07,
|
|
"loss": 0.0138,
|
|
"num_tokens": 48842899.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7263124585151672,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 825,
|
|
"step_time": 43.00261485762894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1633.0,
|
|
"completions/max_terminated_length": 1633.0,
|
|
"completions/mean_length": 721.140625,
|
|
"completions/mean_terminated_length": 721.140625,
|
|
"completions/min_length": 226.0,
|
|
"completions/min_terminated_length": 226.0,
|
|
"entropy": 0.6257605068385601,
|
|
"epoch": 0.46825396825396826,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"kl": 0.01157437771325931,
|
|
"learning_rate": 5.32312925170068e-07,
|
|
"loss": 0.0083,
|
|
"num_tokens": 48898756.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 826,
|
|
"step_time": 57.318738255649805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1300.0,
|
|
"completions/max_terminated_length": 1300.0,
|
|
"completions/mean_length": 676.171875,
|
|
"completions/mean_terminated_length": 676.171875,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.6479384768754244,
|
|
"epoch": 0.4688208616780045,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.640625,
|
|
"kl": 0.03646734420908615,
|
|
"learning_rate": 5.317460317460317e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 48953223.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.8158508539199829,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 827,
|
|
"step_time": 45.83143617399037
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1439.0,
|
|
"completions/max_terminated_length": 1439.0,
|
|
"completions/mean_length": 737.203125,
|
|
"completions/mean_terminated_length": 737.203125,
|
|
"completions/min_length": 252.0,
|
|
"completions/min_terminated_length": 252.0,
|
|
"entropy": 0.8347260430455208,
|
|
"epoch": 0.46938775510204084,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"kl": 0.008276606677100062,
|
|
"learning_rate": 5.311791383219955e-07,
|
|
"loss": -0.0019,
|
|
"num_tokens": 49010476.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 828,
|
|
"step_time": 50.88807295821607
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1258.0,
|
|
"completions/max_terminated_length": 1258.0,
|
|
"completions/mean_length": 825.796875,
|
|
"completions/mean_terminated_length": 825.796875,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.7442977018654346,
|
|
"epoch": 0.4699546485260771,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.005312691297149286,
|
|
"learning_rate": 5.306122448979592e-07,
|
|
"loss": -0.0007,
|
|
"num_tokens": 49076799.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 829,
|
|
"step_time": 45.64249374438077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1027.0,
|
|
"completions/max_terminated_length": 1027.0,
|
|
"completions/mean_length": 638.609375,
|
|
"completions/mean_terminated_length": 638.609375,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.9731990322470665,
|
|
"epoch": 0.47052154195011336,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5390625,
|
|
"kl": 0.015711184809333645,
|
|
"learning_rate": 5.30045351473923e-07,
|
|
"loss": -0.0136,
|
|
"num_tokens": 49131278.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 830,
|
|
"step_time": 39.10309998691082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1659.0,
|
|
"completions/max_terminated_length": 1659.0,
|
|
"completions/mean_length": 826.34375,
|
|
"completions/mean_terminated_length": 826.34375,
|
|
"completions/min_length": 411.0,
|
|
"completions/min_terminated_length": 411.0,
|
|
"entropy": 0.7709367889910936,
|
|
"epoch": 0.4710884353741497,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.008861611728207208,
|
|
"learning_rate": 5.294784580498866e-07,
|
|
"loss": -0.0152,
|
|
"num_tokens": 49193540.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 831,
|
|
"step_time": 58.29900811146945
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1822.0,
|
|
"completions/max_terminated_length": 1822.0,
|
|
"completions/mean_length": 669.78125,
|
|
"completions/mean_terminated_length": 669.78125,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.8484674133360386,
|
|
"epoch": 0.47165532879818595,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.007756266844808124,
|
|
"learning_rate": 5.289115646258502e-07,
|
|
"loss": -0.0172,
|
|
"num_tokens": 49245406.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 832,
|
|
"step_time": 63.82465416099876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1784.0,
|
|
"completions/max_terminated_length": 1784.0,
|
|
"completions/mean_length": 887.75,
|
|
"completions/mean_terminated_length": 887.75,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.6451795902103186,
|
|
"epoch": 0.4722222222222222,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.44921875,
|
|
"kl": 0.00814339253702201,
|
|
"learning_rate": 5.28344671201814e-07,
|
|
"loss": 0.0259,
|
|
"num_tokens": 49311550.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 833,
|
|
"step_time": 62.9865149538964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1561.0,
|
|
"completions/max_terminated_length": 1561.0,
|
|
"completions/mean_length": 691.984375,
|
|
"completions/mean_terminated_length": 691.984375,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 0.8410996180027723,
|
|
"epoch": 0.47278911564625853,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.0159685144899413,
|
|
"learning_rate": 5.277777777777777e-07,
|
|
"loss": -0.0251,
|
|
"num_tokens": 49365965.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 834,
|
|
"step_time": 55.33143498376012
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1623.0,
|
|
"completions/max_terminated_length": 1623.0,
|
|
"completions/mean_length": 791.796875,
|
|
"completions/mean_terminated_length": 791.796875,
|
|
"completions/min_length": 175.0,
|
|
"completions/min_terminated_length": 175.0,
|
|
"entropy": 0.7836950719356537,
|
|
"epoch": 0.4733560090702948,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.013761779322521761,
|
|
"learning_rate": 5.272108843537415e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 49427144.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 835,
|
|
"step_time": 57.657061892561615
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1182.0,
|
|
"completions/max_terminated_length": 1182.0,
|
|
"completions/mean_length": 723.3125,
|
|
"completions/mean_terminated_length": 723.3125,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.7478747200220823,
|
|
"epoch": 0.47392290249433106,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58984375,
|
|
"kl": 0.006380425504175946,
|
|
"learning_rate": 5.266439909297052e-07,
|
|
"loss": 0.0427,
|
|
"num_tokens": 49483276.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 836,
|
|
"step_time": 41.58554752729833
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1417.0,
|
|
"completions/max_terminated_length": 1417.0,
|
|
"completions/mean_length": 741.453125,
|
|
"completions/mean_terminated_length": 741.453125,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.8470651749521494,
|
|
"epoch": 0.4744897959183674,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.008060395397478715,
|
|
"learning_rate": 5.260770975056689e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 49541009.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 837,
|
|
"step_time": 50.17872612923384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1293.0,
|
|
"completions/max_terminated_length": 1293.0,
|
|
"completions/mean_length": 710.0,
|
|
"completions/mean_terminated_length": 710.0,
|
|
"completions/min_length": 316.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"entropy": 0.7036703862249851,
|
|
"epoch": 0.47505668934240364,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6328125,
|
|
"kl": 0.006023438829288352,
|
|
"learning_rate": 5.255102040816326e-07,
|
|
"loss": 0.017,
|
|
"num_tokens": 49595569.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8188852071762085,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 838,
|
|
"step_time": 45.64805651269853
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1186.0,
|
|
"completions/max_terminated_length": 1186.0,
|
|
"completions/mean_length": 671.515625,
|
|
"completions/mean_terminated_length": 671.515625,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.8077564667910337,
|
|
"epoch": 0.4756235827664399,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.00942418300837744,
|
|
"learning_rate": 5.249433106575964e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 49649834.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6893463730812073,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 839,
|
|
"step_time": 42.007688380777836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1114.0,
|
|
"completions/max_terminated_length": 1114.0,
|
|
"completions/mean_length": 680.890625,
|
|
"completions/mean_terminated_length": 680.890625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.9049110170453787,
|
|
"epoch": 0.47619047619047616,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.014862482465105131,
|
|
"learning_rate": 5.2437641723356e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 49706211.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 840,
|
|
"step_time": 41.27778664045036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1320.0,
|
|
"completions/max_terminated_length": 1320.0,
|
|
"completions/mean_length": 749.75,
|
|
"completions/mean_terminated_length": 749.75,
|
|
"completions/min_length": 232.0,
|
|
"completions/min_terminated_length": 232.0,
|
|
"entropy": 0.7302116099745035,
|
|
"epoch": 0.4767573696145125,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"kl": 0.010044565831776708,
|
|
"learning_rate": 5.238095238095238e-07,
|
|
"loss": -0.0228,
|
|
"num_tokens": 49764747.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6993034482002258,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 841,
|
|
"step_time": 46.80342169199139
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1509.0,
|
|
"completions/max_terminated_length": 1509.0,
|
|
"completions/mean_length": 795.984375,
|
|
"completions/mean_terminated_length": 795.984375,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 0.8341542407870293,
|
|
"epoch": 0.47732426303854875,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.271484375,
|
|
"kl": 0.00599672862153966,
|
|
"learning_rate": 5.232426303854875e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 49827066.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 842,
|
|
"step_time": 54.41800384223461
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1584.0,
|
|
"completions/max_terminated_length": 1584.0,
|
|
"completions/mean_length": 702.84375,
|
|
"completions/mean_terminated_length": 702.84375,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.6113820169121027,
|
|
"epoch": 0.477891156462585,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.00742972623265814,
|
|
"learning_rate": 5.226757369614512e-07,
|
|
"loss": -0.0143,
|
|
"num_tokens": 49881064.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 843,
|
|
"step_time": 55.46922871749848
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1215.0,
|
|
"completions/max_terminated_length": 1215.0,
|
|
"completions/mean_length": 698.09375,
|
|
"completions/mean_terminated_length": 698.09375,
|
|
"completions/min_length": 257.0,
|
|
"completions/min_terminated_length": 257.0,
|
|
"entropy": 0.9045372642576694,
|
|
"epoch": 0.47845804988662133,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.013145714270649478,
|
|
"learning_rate": 5.221088435374149e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 49935302.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 844,
|
|
"step_time": 42.87468405812979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1395.0,
|
|
"completions/max_terminated_length": 1395.0,
|
|
"completions/mean_length": 783.328125,
|
|
"completions/mean_terminated_length": 783.328125,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 0.7101977486163378,
|
|
"epoch": 0.4790249433106576,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.439453125,
|
|
"kl": 0.00962134356086608,
|
|
"learning_rate": 5.215419501133787e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 49997803.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 845,
|
|
"step_time": 49.764672949910164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1182.0,
|
|
"completions/max_terminated_length": 1182.0,
|
|
"completions/mean_length": 679.109375,
|
|
"completions/mean_terminated_length": 679.109375,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 0.9777658823877573,
|
|
"epoch": 0.47959183673469385,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.008945881316321902,
|
|
"learning_rate": 5.209750566893424e-07,
|
|
"loss": 0.0143,
|
|
"num_tokens": 50050922.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 846,
|
|
"step_time": 41.81044256687164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1949.0,
|
|
"completions/max_terminated_length": 1949.0,
|
|
"completions/mean_length": 833.234375,
|
|
"completions/mean_terminated_length": 833.234375,
|
|
"completions/min_length": 229.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"entropy": 0.616375869140029,
|
|
"epoch": 0.4801587301587302,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.0144760866969591,
|
|
"learning_rate": 5.204081632653062e-07,
|
|
"loss": 0.0224,
|
|
"num_tokens": 50114121.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 847,
|
|
"step_time": 69.3843470979482
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1574.0,
|
|
"completions/max_terminated_length": 1574.0,
|
|
"completions/mean_length": 834.578125,
|
|
"completions/mean_terminated_length": 834.578125,
|
|
"completions/min_length": 217.0,
|
|
"completions/min_terminated_length": 217.0,
|
|
"entropy": 0.6404037345200777,
|
|
"epoch": 0.48072562358276644,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.0073050470382440835,
|
|
"learning_rate": 5.198412698412698e-07,
|
|
"loss": 0.0196,
|
|
"num_tokens": 50176854.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 848,
|
|
"step_time": 55.65150818694383
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1558.0,
|
|
"completions/max_terminated_length": 1558.0,
|
|
"completions/mean_length": 641.28125,
|
|
"completions/mean_terminated_length": 641.28125,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 0.8769101612269878,
|
|
"epoch": 0.4812925170068027,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.014476951531833038,
|
|
"learning_rate": 5.192743764172335e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 50228896.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 849,
|
|
"step_time": 55.60157973412424
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1110.0,
|
|
"completions/max_terminated_length": 1110.0,
|
|
"completions/mean_length": 654.75,
|
|
"completions/mean_terminated_length": 654.75,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.6942555997520685,
|
|
"epoch": 0.481859410430839,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.015545000162092037,
|
|
"learning_rate": 5.187074829931972e-07,
|
|
"loss": -0.0122,
|
|
"num_tokens": 50281264.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 850,
|
|
"step_time": 40.07644457463175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1341.0,
|
|
"completions/max_terminated_length": 1341.0,
|
|
"completions/mean_length": 736.09375,
|
|
"completions/mean_terminated_length": 736.09375,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 0.8285705111920834,
|
|
"epoch": 0.4824263038548753,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2080078125,
|
|
"kl": 0.009163045207969844,
|
|
"learning_rate": 5.18140589569161e-07,
|
|
"loss": 0.0024,
|
|
"num_tokens": 50338910.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 851,
|
|
"step_time": 47.86978491861373
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1022.0,
|
|
"completions/max_terminated_length": 1022.0,
|
|
"completions/mean_length": 664.84375,
|
|
"completions/mean_terminated_length": 664.84375,
|
|
"completions/min_length": 275.0,
|
|
"completions/min_terminated_length": 275.0,
|
|
"entropy": 1.1474276669323444,
|
|
"epoch": 0.48299319727891155,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.009676028945250437,
|
|
"learning_rate": 5.175736961451247e-07,
|
|
"loss": -0.0009,
|
|
"num_tokens": 50391764.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 852,
|
|
"step_time": 36.22214914299548
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1430.0,
|
|
"completions/max_terminated_length": 1430.0,
|
|
"completions/mean_length": 729.328125,
|
|
"completions/mean_terminated_length": 729.328125,
|
|
"completions/min_length": 382.0,
|
|
"completions/min_terminated_length": 382.0,
|
|
"entropy": 0.8884090799838305,
|
|
"epoch": 0.48356009070294786,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.007700895963353105,
|
|
"learning_rate": 5.170068027210885e-07,
|
|
"loss": -0.0207,
|
|
"num_tokens": 50448937.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7911965847015381,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 853,
|
|
"step_time": 51.08947387896478
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1355.0,
|
|
"completions/max_terminated_length": 1355.0,
|
|
"completions/mean_length": 757.140625,
|
|
"completions/mean_terminated_length": 757.140625,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.9926306642591953,
|
|
"epoch": 0.48412698412698413,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.009952055668691173,
|
|
"learning_rate": 5.164399092970522e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 50509050.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 854,
|
|
"step_time": 48.03361593559384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1820.0,
|
|
"completions/max_terminated_length": 1820.0,
|
|
"completions/mean_length": 870.015625,
|
|
"completions/mean_terminated_length": 870.015625,
|
|
"completions/min_length": 416.0,
|
|
"completions/min_terminated_length": 416.0,
|
|
"entropy": 0.6667082589119673,
|
|
"epoch": 0.4846938775510204,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3203125,
|
|
"kl": 0.012756725016515702,
|
|
"learning_rate": 5.15873015873016e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 50575291.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 855,
|
|
"step_time": 64.74019164126366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1655.0,
|
|
"completions/max_terminated_length": 1655.0,
|
|
"completions/mean_length": 665.890625,
|
|
"completions/mean_terminated_length": 665.890625,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.8283300343900919,
|
|
"epoch": 0.4852607709750567,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.006617442209972069,
|
|
"learning_rate": 5.153061224489795e-07,
|
|
"loss": 0.0208,
|
|
"num_tokens": 50627996.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 856,
|
|
"step_time": 58.02800523210317
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 998.0,
|
|
"completions/max_terminated_length": 998.0,
|
|
"completions/mean_length": 603.34375,
|
|
"completions/mean_terminated_length": 603.34375,
|
|
"completions/min_length": 211.0,
|
|
"completions/min_terminated_length": 211.0,
|
|
"entropy": 0.7314907535910606,
|
|
"epoch": 0.485827664399093,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.011034592549549416,
|
|
"learning_rate": 5.147392290249432e-07,
|
|
"loss": -0.0177,
|
|
"num_tokens": 50678098.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.7983350157737732,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 857,
|
|
"step_time": 35.60801274795085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1401.0,
|
|
"completions/max_terminated_length": 1401.0,
|
|
"completions/mean_length": 799.1875,
|
|
"completions/mean_terminated_length": 799.1875,
|
|
"completions/min_length": 472.0,
|
|
"completions/min_terminated_length": 472.0,
|
|
"entropy": 0.7935092113912106,
|
|
"epoch": 0.48639455782312924,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.012917440486489795,
|
|
"learning_rate": 5.14172335600907e-07,
|
|
"loss": -0.0119,
|
|
"num_tokens": 50740534.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 858,
|
|
"step_time": 50.11415920685977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1592.0,
|
|
"completions/max_terminated_length": 1592.0,
|
|
"completions/mean_length": 718.203125,
|
|
"completions/mean_terminated_length": 718.203125,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.9313573502004147,
|
|
"epoch": 0.48696145124716556,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40625,
|
|
"kl": 0.0080044867208926,
|
|
"learning_rate": 5.136054421768707e-07,
|
|
"loss": -0.0117,
|
|
"num_tokens": 50796395.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 859,
|
|
"step_time": 55.48327258042991
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1165.0,
|
|
"completions/max_terminated_length": 1165.0,
|
|
"completions/mean_length": 675.5625,
|
|
"completions/mean_terminated_length": 675.5625,
|
|
"completions/min_length": 220.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"entropy": 0.8820345290005207,
|
|
"epoch": 0.4875283446712018,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.032954859620076604,
|
|
"learning_rate": 5.130385487528345e-07,
|
|
"loss": -0.0028,
|
|
"num_tokens": 50849119.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 860,
|
|
"step_time": 41.23665721621364
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1324.0,
|
|
"completions/max_terminated_length": 1324.0,
|
|
"completions/mean_length": 745.9375,
|
|
"completions/mean_terminated_length": 745.9375,
|
|
"completions/min_length": 401.0,
|
|
"completions/min_terminated_length": 401.0,
|
|
"entropy": 0.6874312870204449,
|
|
"epoch": 0.4880952380952381,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3515625,
|
|
"kl": 0.007448262127581984,
|
|
"learning_rate": 5.124716553287982e-07,
|
|
"loss": 0.0157,
|
|
"num_tokens": 50907547.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 861,
|
|
"step_time": 46.97997191455215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1752.0,
|
|
"completions/mean_length": 822.578125,
|
|
"completions/mean_terminated_length": 770.6190795898438,
|
|
"completions/min_length": 212.0,
|
|
"completions/min_terminated_length": 212.0,
|
|
"entropy": 0.6749304560944438,
|
|
"epoch": 0.4886621315192744,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.369140625,
|
|
"kl": 0.005468021234264597,
|
|
"learning_rate": 5.119047619047619e-07,
|
|
"loss": -0.0592,
|
|
"num_tokens": 50970096.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 862,
|
|
"step_time": 171.0216311365366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1239.0,
|
|
"completions/max_terminated_length": 1239.0,
|
|
"completions/mean_length": 594.578125,
|
|
"completions/mean_terminated_length": 594.578125,
|
|
"completions/min_length": 205.0,
|
|
"completions/min_terminated_length": 205.0,
|
|
"entropy": 0.7492969539016485,
|
|
"epoch": 0.48922902494331066,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7265625,
|
|
"kl": 0.00867966354417149,
|
|
"learning_rate": 5.113378684807256e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 51018253.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.7983350157737732,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 863,
|
|
"step_time": 43.79116830602288
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1055.0,
|
|
"completions/max_terminated_length": 1055.0,
|
|
"completions/mean_length": 629.359375,
|
|
"completions/mean_terminated_length": 629.359375,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 1.1514921709895134,
|
|
"epoch": 0.4897959183673469,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.8125,
|
|
"kl": 0.008485040816594847,
|
|
"learning_rate": 5.107709750566893e-07,
|
|
"loss": 0.0226,
|
|
"num_tokens": 51070212.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 864,
|
|
"step_time": 38.110903449356556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1564.0,
|
|
"completions/max_terminated_length": 1564.0,
|
|
"completions/mean_length": 789.0,
|
|
"completions/mean_terminated_length": 789.0,
|
|
"completions/min_length": 454.0,
|
|
"completions/min_terminated_length": 454.0,
|
|
"entropy": 0.7437616940587759,
|
|
"epoch": 0.49036281179138325,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.01059547925251536,
|
|
"learning_rate": 5.10204081632653e-07,
|
|
"loss": 0.0059,
|
|
"num_tokens": 51131532.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 865,
|
|
"step_time": 55.38527577649802
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1498.0,
|
|
"completions/max_terminated_length": 1498.0,
|
|
"completions/mean_length": 694.046875,
|
|
"completions/mean_terminated_length": 694.046875,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 1.0644450411200523,
|
|
"epoch": 0.4909297052154195,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.025005857984069735,
|
|
"learning_rate": 5.096371882086168e-07,
|
|
"loss": -0.0054,
|
|
"num_tokens": 51187279.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 866,
|
|
"step_time": 53.09744997229427
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1217.0,
|
|
"completions/max_terminated_length": 1217.0,
|
|
"completions/mean_length": 724.9375,
|
|
"completions/mean_terminated_length": 724.9375,
|
|
"completions/min_length": 424.0,
|
|
"completions/min_terminated_length": 424.0,
|
|
"entropy": 0.8717704601585865,
|
|
"epoch": 0.4914965986394558,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.013817924278555438,
|
|
"learning_rate": 5.090702947845805e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 51249659.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 867,
|
|
"step_time": 44.83446109481156
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1772.0,
|
|
"completions/max_terminated_length": 1772.0,
|
|
"completions/mean_length": 736.46875,
|
|
"completions/mean_terminated_length": 736.46875,
|
|
"completions/min_length": 365.0,
|
|
"completions/min_terminated_length": 365.0,
|
|
"entropy": 0.707472387701273,
|
|
"epoch": 0.49206349206349204,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.008129013964207843,
|
|
"learning_rate": 5.085034013605442e-07,
|
|
"loss": 0.0399,
|
|
"num_tokens": 51306497.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 868,
|
|
"step_time": 63.36467006895691
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1363.0,
|
|
"completions/max_terminated_length": 1363.0,
|
|
"completions/mean_length": 753.328125,
|
|
"completions/mean_terminated_length": 753.328125,
|
|
"completions/min_length": 134.0,
|
|
"completions/min_terminated_length": 134.0,
|
|
"entropy": 0.7304238229990005,
|
|
"epoch": 0.49263038548752835,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.007328970619710162,
|
|
"learning_rate": 5.079365079365079e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 51364022.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 869,
|
|
"step_time": 47.95321795810014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1728.0,
|
|
"completions/max_terminated_length": 1728.0,
|
|
"completions/mean_length": 807.640625,
|
|
"completions/mean_terminated_length": 807.640625,
|
|
"completions/min_length": 458.0,
|
|
"completions/min_terminated_length": 458.0,
|
|
"entropy": 0.866294052451849,
|
|
"epoch": 0.4931972789115646,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.006608701121876948,
|
|
"learning_rate": 5.073696145124717e-07,
|
|
"loss": 0.0199,
|
|
"num_tokens": 51427735.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 870,
|
|
"step_time": 61.613203305751085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1934.0,
|
|
"completions/max_terminated_length": 1934.0,
|
|
"completions/mean_length": 748.5625,
|
|
"completions/mean_terminated_length": 748.5625,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.816673843190074,
|
|
"epoch": 0.4937641723356009,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.005822037754114717,
|
|
"learning_rate": 5.068027210884354e-07,
|
|
"loss": -0.0257,
|
|
"num_tokens": 51486795.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 871,
|
|
"step_time": 69.48857913352549
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1560.0,
|
|
"completions/max_terminated_length": 1560.0,
|
|
"completions/mean_length": 754.296875,
|
|
"completions/mean_terminated_length": 754.296875,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"entropy": 0.601695166900754,
|
|
"epoch": 0.4943310657596372,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.01173756632488221,
|
|
"learning_rate": 5.062358276643991e-07,
|
|
"loss": 0.0146,
|
|
"num_tokens": 51544230.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 872,
|
|
"step_time": 54.63390828575939
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2051.0,
|
|
"completions/max_terminated_length": 2051.0,
|
|
"completions/mean_length": 716.734375,
|
|
"completions/mean_terminated_length": 716.734375,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.9431626871228218,
|
|
"epoch": 0.49489795918367346,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.017384584236424416,
|
|
"learning_rate": 5.056689342403628e-07,
|
|
"loss": -0.0066,
|
|
"num_tokens": 51600821.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 873,
|
|
"step_time": 73.22143149282783
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2295.0,
|
|
"completions/max_terminated_length": 2295.0,
|
|
"completions/mean_length": 798.84375,
|
|
"completions/mean_terminated_length": 798.84375,
|
|
"completions/min_length": 407.0,
|
|
"completions/min_terminated_length": 407.0,
|
|
"entropy": 0.9745409674942493,
|
|
"epoch": 0.4954648526077097,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"kl": 0.008309980170452036,
|
|
"learning_rate": 5.051020408163264e-07,
|
|
"loss": 0.0293,
|
|
"num_tokens": 51665155.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 874,
|
|
"step_time": 83.59826904442161
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1261.0,
|
|
"completions/max_terminated_length": 1261.0,
|
|
"completions/mean_length": 637.5,
|
|
"completions/mean_terminated_length": 637.5,
|
|
"completions/min_length": 164.0,
|
|
"completions/min_terminated_length": 164.0,
|
|
"entropy": 1.0830895081162453,
|
|
"epoch": 0.49603174603174605,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.007421919493936002,
|
|
"learning_rate": 5.045351473922902e-07,
|
|
"loss": -0.0059,
|
|
"num_tokens": 51725011.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 875,
|
|
"step_time": 50.99143857136369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1361.0,
|
|
"completions/max_terminated_length": 1361.0,
|
|
"completions/mean_length": 673.421875,
|
|
"completions/mean_terminated_length": 673.421875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.8804706893861294,
|
|
"epoch": 0.4965986394557823,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.011465668474556878,
|
|
"learning_rate": 5.039682539682539e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 51778670.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6878606677055359,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 876,
|
|
"step_time": 48.2349415756762
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 738.890625,
|
|
"completions/mean_terminated_length": 738.890625,
|
|
"completions/min_length": 365.0,
|
|
"completions/min_terminated_length": 365.0,
|
|
"entropy": 0.8532643038779497,
|
|
"epoch": 0.4971655328798186,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.357421875,
|
|
"kl": 0.008807096746750176,
|
|
"learning_rate": 5.034013605442177e-07,
|
|
"loss": 0.0147,
|
|
"num_tokens": 51838823.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 877,
|
|
"step_time": 50.15280640684068
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1499.0,
|
|
"completions/max_terminated_length": 1499.0,
|
|
"completions/mean_length": 679.0,
|
|
"completions/mean_terminated_length": 679.0,
|
|
"completions/min_length": 172.0,
|
|
"completions/min_terminated_length": 172.0,
|
|
"entropy": 0.8594838082790375,
|
|
"epoch": 0.4977324263038549,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.73828125,
|
|
"kl": 0.020510786271188408,
|
|
"learning_rate": 5.028344671201814e-07,
|
|
"loss": 0.0108,
|
|
"num_tokens": 51892383.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 878,
|
|
"step_time": 52.52436824142933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1390.0,
|
|
"completions/max_terminated_length": 1390.0,
|
|
"completions/mean_length": 750.1875,
|
|
"completions/mean_terminated_length": 750.1875,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.8799360301345587,
|
|
"epoch": 0.49829931972789115,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.010521399381104857,
|
|
"learning_rate": 5.022675736961452e-07,
|
|
"loss": 0.0011,
|
|
"num_tokens": 51951531.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 879,
|
|
"step_time": 49.18503315653652
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2307.0,
|
|
"completions/max_terminated_length": 2307.0,
|
|
"completions/mean_length": 783.5,
|
|
"completions/mean_terminated_length": 783.5,
|
|
"completions/min_length": 178.0,
|
|
"completions/min_terminated_length": 178.0,
|
|
"entropy": 0.6585966069251299,
|
|
"epoch": 0.4988662131519274,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.006630451563978568,
|
|
"learning_rate": 5.017006802721087e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 52011579.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 880,
|
|
"step_time": 83.87827741727233
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1611.0,
|
|
"completions/max_terminated_length": 1611.0,
|
|
"completions/mean_length": 769.03125,
|
|
"completions/mean_terminated_length": 769.03125,
|
|
"completions/min_length": 42.0,
|
|
"completions/min_terminated_length": 42.0,
|
|
"entropy": 0.8288990035653114,
|
|
"epoch": 0.49943310657596374,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"kl": 0.007872988499002531,
|
|
"learning_rate": 5.011337868480725e-07,
|
|
"loss": 0.0073,
|
|
"num_tokens": 52070357.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 881,
|
|
"step_time": 56.92433960083872
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1323.0,
|
|
"completions/max_terminated_length": 1323.0,
|
|
"completions/mean_length": 686.828125,
|
|
"completions/mean_terminated_length": 686.828125,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 0.8330890834331512,
|
|
"epoch": 0.5,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4765625,
|
|
"kl": 0.00948665602481924,
|
|
"learning_rate": 5.005668934240362e-07,
|
|
"loss": -0.0257,
|
|
"num_tokens": 52124154.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 882,
|
|
"step_time": 47.21040133293718
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1570.0,
|
|
"completions/max_terminated_length": 1570.0,
|
|
"completions/mean_length": 842.921875,
|
|
"completions/mean_terminated_length": 842.921875,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.6781508959829807,
|
|
"epoch": 0.5005668934240363,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.220703125,
|
|
"kl": 0.007487549592042342,
|
|
"learning_rate": 5e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 52188773.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 883,
|
|
"step_time": 55.30087985098362
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1552.0,
|
|
"completions/max_terminated_length": 1552.0,
|
|
"completions/mean_length": 646.453125,
|
|
"completions/mean_terminated_length": 646.453125,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 0.7466154173016548,
|
|
"epoch": 0.5011337868480725,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6640625,
|
|
"kl": 0.008228908074670471,
|
|
"learning_rate": 4.994331065759637e-07,
|
|
"loss": 0.0247,
|
|
"num_tokens": 52239282.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.7759642601013184,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 884,
|
|
"step_time": 54.73128490615636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1212.0,
|
|
"completions/max_terminated_length": 1212.0,
|
|
"completions/mean_length": 741.125,
|
|
"completions/mean_terminated_length": 741.125,
|
|
"completions/min_length": 347.0,
|
|
"completions/min_terminated_length": 347.0,
|
|
"entropy": 0.9275486841797829,
|
|
"epoch": 0.5017006802721088,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.236328125,
|
|
"kl": 0.008196077920729294,
|
|
"learning_rate": 4.988662131519274e-07,
|
|
"loss": 0.0049,
|
|
"num_tokens": 52298754.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 885,
|
|
"step_time": 43.7424898929894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1473.0,
|
|
"completions/max_terminated_length": 1473.0,
|
|
"completions/mean_length": 751.78125,
|
|
"completions/mean_terminated_length": 751.78125,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.8345377556979656,
|
|
"epoch": 0.5022675736961452,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.012016213266178966,
|
|
"learning_rate": 4.982993197278911e-07,
|
|
"loss": -0.0046,
|
|
"num_tokens": 52356956.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 886,
|
|
"step_time": 51.94546024128795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1351.0,
|
|
"completions/max_terminated_length": 1351.0,
|
|
"completions/mean_length": 778.828125,
|
|
"completions/mean_terminated_length": 778.828125,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.6318278256803751,
|
|
"epoch": 0.5028344671201814,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.013434714026516303,
|
|
"learning_rate": 4.977324263038549e-07,
|
|
"loss": 0.0333,
|
|
"num_tokens": 52419809.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 887,
|
|
"step_time": 48.086403116583824
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1560.0,
|
|
"completions/max_terminated_length": 1560.0,
|
|
"completions/mean_length": 719.9375,
|
|
"completions/mean_terminated_length": 719.9375,
|
|
"completions/min_length": 189.0,
|
|
"completions/min_terminated_length": 189.0,
|
|
"entropy": 0.691303588449955,
|
|
"epoch": 0.5034013605442177,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.008749918502871878,
|
|
"learning_rate": 4.971655328798185e-07,
|
|
"loss": 0.015,
|
|
"num_tokens": 52474437.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 888,
|
|
"step_time": 54.65371115040034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1512.0,
|
|
"completions/max_terminated_length": 1512.0,
|
|
"completions/mean_length": 647.546875,
|
|
"completions/mean_terminated_length": 647.546875,
|
|
"completions/min_length": 259.0,
|
|
"completions/min_terminated_length": 259.0,
|
|
"entropy": 0.758992325514555,
|
|
"epoch": 0.503968253968254,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.62109375,
|
|
"kl": 0.009968082042178139,
|
|
"learning_rate": 4.965986394557823e-07,
|
|
"loss": 0.0188,
|
|
"num_tokens": 52527176.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6542273163795471,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 889,
|
|
"step_time": 53.365698255598545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1424.0,
|
|
"completions/max_terminated_length": 1424.0,
|
|
"completions/mean_length": 821.859375,
|
|
"completions/mean_terminated_length": 821.859375,
|
|
"completions/min_length": 488.0,
|
|
"completions/min_terminated_length": 488.0,
|
|
"entropy": 0.9987582489848137,
|
|
"epoch": 0.5045351473922902,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.013309484973433428,
|
|
"learning_rate": 4.96031746031746e-07,
|
|
"loss": -0.0231,
|
|
"num_tokens": 52592087.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 890,
|
|
"step_time": 51.339532599784434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1276.0,
|
|
"completions/max_terminated_length": 1276.0,
|
|
"completions/mean_length": 726.8125,
|
|
"completions/mean_terminated_length": 726.8125,
|
|
"completions/min_length": 243.0,
|
|
"completions/min_terminated_length": 243.0,
|
|
"entropy": 0.7470531817525625,
|
|
"epoch": 0.5051020408163265,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.01297632603382226,
|
|
"learning_rate": 4.954648526077097e-07,
|
|
"loss": -0.0006,
|
|
"num_tokens": 52647803.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 891,
|
|
"step_time": 45.2211769586429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1608.0,
|
|
"completions/max_terminated_length": 1608.0,
|
|
"completions/mean_length": 748.25,
|
|
"completions/mean_terminated_length": 748.25,
|
|
"completions/min_length": 386.0,
|
|
"completions/min_terminated_length": 386.0,
|
|
"entropy": 1.0034601390361786,
|
|
"epoch": 0.5056689342403629,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.306640625,
|
|
"kl": 0.010633844882249832,
|
|
"learning_rate": 4.948979591836734e-07,
|
|
"loss": 0.03,
|
|
"num_tokens": 52707283.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5107961297035217,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 892,
|
|
"step_time": 57.29739475250244
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1421.0,
|
|
"completions/max_terminated_length": 1421.0,
|
|
"completions/mean_length": 705.4375,
|
|
"completions/mean_terminated_length": 705.4375,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.6611053962260485,
|
|
"epoch": 0.5062358276643991,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.01157246605725959,
|
|
"learning_rate": 4.943310657596372e-07,
|
|
"loss": 0.0012,
|
|
"num_tokens": 52761767.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7263124585151672,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 893,
|
|
"step_time": 50.51417244318873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1385.0,
|
|
"completions/max_terminated_length": 1385.0,
|
|
"completions/mean_length": 741.109375,
|
|
"completions/mean_terminated_length": 741.109375,
|
|
"completions/min_length": 399.0,
|
|
"completions/min_terminated_length": 399.0,
|
|
"entropy": 0.8032779544591904,
|
|
"epoch": 0.5068027210884354,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.671875,
|
|
"kl": 0.008927945280447602,
|
|
"learning_rate": 4.937641723356009e-07,
|
|
"loss": -0.0181,
|
|
"num_tokens": 52819478.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7559289336204529,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 894,
|
|
"step_time": 48.87483713682741
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3296.0,
|
|
"completions/max_terminated_length": 3296.0,
|
|
"completions/mean_length": 738.75,
|
|
"completions/mean_terminated_length": 738.75,
|
|
"completions/min_length": 340.0,
|
|
"completions/min_terminated_length": 340.0,
|
|
"entropy": 0.7655461337417364,
|
|
"epoch": 0.5073696145124716,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.009647198297898285,
|
|
"learning_rate": 4.931972789115645e-07,
|
|
"loss": 0.0955,
|
|
"num_tokens": 52877270.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 895,
|
|
"step_time": 128.44080637115985
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1305.0,
|
|
"completions/max_terminated_length": 1305.0,
|
|
"completions/mean_length": 658.15625,
|
|
"completions/mean_terminated_length": 658.15625,
|
|
"completions/min_length": 144.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"entropy": 0.47649334371089935,
|
|
"epoch": 0.5079365079365079,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.01859457766113337,
|
|
"learning_rate": 4.926303854875283e-07,
|
|
"loss": -0.029,
|
|
"num_tokens": 52931152.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.8158508539199829,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 896,
|
|
"step_time": 47.568701623007655
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1362.0,
|
|
"completions/max_terminated_length": 1362.0,
|
|
"completions/mean_length": 696.046875,
|
|
"completions/mean_terminated_length": 696.046875,
|
|
"completions/min_length": 275.0,
|
|
"completions/min_terminated_length": 275.0,
|
|
"entropy": 0.7470553796738386,
|
|
"epoch": 0.5085034013605442,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.00784090360684786,
|
|
"learning_rate": 4.92063492063492e-07,
|
|
"loss": -0.0132,
|
|
"num_tokens": 52986451.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7911965847015381,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 897,
|
|
"step_time": 48.28644000273198
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1381.0,
|
|
"completions/max_terminated_length": 1381.0,
|
|
"completions/mean_length": 770.75,
|
|
"completions/mean_terminated_length": 770.75,
|
|
"completions/min_length": 338.0,
|
|
"completions/min_terminated_length": 338.0,
|
|
"entropy": 0.76389548368752,
|
|
"epoch": 0.5090702947845805,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.012054729741066694,
|
|
"learning_rate": 4.914965986394558e-07,
|
|
"loss": -0.0199,
|
|
"num_tokens": 53045955.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6986381411552429,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 898,
|
|
"step_time": 48.94874201063067
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/max_terminated_length": 1376.0,
|
|
"completions/mean_length": 714.015625,
|
|
"completions/mean_terminated_length": 714.015625,
|
|
"completions/min_length": 264.0,
|
|
"completions/min_terminated_length": 264.0,
|
|
"entropy": 0.8200095966458321,
|
|
"epoch": 0.5096371882086168,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.010402989326394163,
|
|
"learning_rate": 4.909297052154194e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 53100164.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.7126966714859009,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 899,
|
|
"step_time": 48.38518248870969
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1282.0,
|
|
"completions/max_terminated_length": 1282.0,
|
|
"completions/mean_length": 812.921875,
|
|
"completions/mean_terminated_length": 812.921875,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.7548295799642801,
|
|
"epoch": 0.5102040816326531,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.005744386944570579,
|
|
"learning_rate": 4.903628117913832e-07,
|
|
"loss": 0.0173,
|
|
"num_tokens": 53162311.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.701074481010437,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 900,
|
|
"step_time": 45.458668146282434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1300.0,
|
|
"completions/max_terminated_length": 1300.0,
|
|
"completions/mean_length": 664.53125,
|
|
"completions/mean_terminated_length": 664.53125,
|
|
"completions/min_length": 213.0,
|
|
"completions/min_terminated_length": 213.0,
|
|
"entropy": 0.6737108416855335,
|
|
"epoch": 0.5107709750566893,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7265625,
|
|
"kl": 0.015609543421305716,
|
|
"learning_rate": 4.897959183673469e-07,
|
|
"loss": 0.0315,
|
|
"num_tokens": 53214449.0,
|
|
"reward": 0.6328125,
|
|
"reward_std": 0.9225662350654602,
|
|
"rewards/accuracy_reward_func/mean": 0.59375,
|
|
"rewards/accuracy_reward_func/std": 0.9209855198860168,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 901,
|
|
"step_time": 45.324661147780716
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1642.0,
|
|
"completions/max_terminated_length": 1642.0,
|
|
"completions/mean_length": 813.78125,
|
|
"completions/mean_terminated_length": 813.78125,
|
|
"completions/min_length": 307.0,
|
|
"completions/min_terminated_length": 307.0,
|
|
"entropy": 0.7072150576859713,
|
|
"epoch": 0.5113378684807256,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.006149703782284632,
|
|
"learning_rate": 4.892290249433107e-07,
|
|
"loss": 0.0291,
|
|
"num_tokens": 53275915.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6782841682434082,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 902,
|
|
"step_time": 57.81173918955028
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1222.0,
|
|
"completions/max_terminated_length": 1222.0,
|
|
"completions/mean_length": 770.734375,
|
|
"completions/mean_terminated_length": 770.734375,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.7643672861158848,
|
|
"epoch": 0.5119047619047619,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6796875,
|
|
"kl": 0.009236391197191551,
|
|
"learning_rate": 4.886621315192743e-07,
|
|
"loss": -0.028,
|
|
"num_tokens": 53336178.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8188852071762085,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 903,
|
|
"step_time": 43.175291545689106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1600.0,
|
|
"completions/max_terminated_length": 1600.0,
|
|
"completions/mean_length": 816.46875,
|
|
"completions/mean_terminated_length": 816.46875,
|
|
"completions/min_length": 390.0,
|
|
"completions/min_terminated_length": 390.0,
|
|
"entropy": 0.8198769446462393,
|
|
"epoch": 0.5124716553287982,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.006929851995664649,
|
|
"learning_rate": 4.880952380952381e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 53399736.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.774084210395813,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 904,
|
|
"step_time": 56.71235060598701
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1507.0,
|
|
"completions/max_terminated_length": 1507.0,
|
|
"completions/mean_length": 769.46875,
|
|
"completions/mean_terminated_length": 769.46875,
|
|
"completions/min_length": 322.0,
|
|
"completions/min_terminated_length": 322.0,
|
|
"entropy": 0.9897831846028566,
|
|
"epoch": 0.5130385487528345,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.009972095809644088,
|
|
"learning_rate": 4.875283446712018e-07,
|
|
"loss": 0.0319,
|
|
"num_tokens": 53462718.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 905,
|
|
"step_time": 53.80057194083929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1392.0,
|
|
"completions/max_terminated_length": 1392.0,
|
|
"completions/mean_length": 671.9375,
|
|
"completions/mean_terminated_length": 671.9375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.8497813642024994,
|
|
"epoch": 0.5136054421768708,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.017676412215223536,
|
|
"learning_rate": 4.869614512471656e-07,
|
|
"loss": -0.001,
|
|
"num_tokens": 53516298.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 906,
|
|
"step_time": 49.45692455023527
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1728.0,
|
|
"completions/max_terminated_length": 1728.0,
|
|
"completions/mean_length": 850.453125,
|
|
"completions/mean_terminated_length": 850.453125,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.6535182613879442,
|
|
"epoch": 0.514172335600907,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.009574678493663669,
|
|
"learning_rate": 4.863945578231292e-07,
|
|
"loss": -0.0057,
|
|
"num_tokens": 53580551.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 907,
|
|
"step_time": 61.371295465156436
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1167.0,
|
|
"completions/max_terminated_length": 1167.0,
|
|
"completions/mean_length": 696.375,
|
|
"completions/mean_terminated_length": 696.375,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.8291148729622364,
|
|
"epoch": 0.5147392290249433,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.013869064976461232,
|
|
"learning_rate": 4.85827664399093e-07,
|
|
"loss": -0.0114,
|
|
"num_tokens": 53637551.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4734388589859009,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 908,
|
|
"step_time": 42.341789598576725
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 4005.0,
|
|
"completions/max_terminated_length": 4005.0,
|
|
"completions/mean_length": 782.609375,
|
|
"completions/mean_terminated_length": 782.609375,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"entropy": 0.6535202991217375,
|
|
"epoch": 0.5153061224489796,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.007209262577816844,
|
|
"learning_rate": 4.852607709750567e-07,
|
|
"loss": 0.0357,
|
|
"num_tokens": 53699830.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 909,
|
|
"step_time": 170.31518048606813
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1588.0,
|
|
"completions/max_terminated_length": 1588.0,
|
|
"completions/mean_length": 854.796875,
|
|
"completions/mean_terminated_length": 854.796875,
|
|
"completions/min_length": 324.0,
|
|
"completions/min_terminated_length": 324.0,
|
|
"entropy": 0.7914769724011421,
|
|
"epoch": 0.5158730158730159,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.0044787218066630885,
|
|
"learning_rate": 4.846938775510204e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 53767585.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 910,
|
|
"step_time": 56.55893933400512
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1529.0,
|
|
"completions/max_terminated_length": 1529.0,
|
|
"completions/mean_length": 763.1875,
|
|
"completions/mean_terminated_length": 763.1875,
|
|
"completions/min_length": 330.0,
|
|
"completions/min_terminated_length": 330.0,
|
|
"entropy": 0.76618417724967,
|
|
"epoch": 0.5164399092970522,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.009190422715619206,
|
|
"learning_rate": 4.841269841269841e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 53824773.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 911,
|
|
"step_time": 53.94998424965888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1555.0,
|
|
"completions/max_terminated_length": 1555.0,
|
|
"completions/mean_length": 728.53125,
|
|
"completions/mean_terminated_length": 728.53125,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.9757031872868538,
|
|
"epoch": 0.5170068027210885,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.455078125,
|
|
"kl": 0.007862636717618443,
|
|
"learning_rate": 4.835600907029479e-07,
|
|
"loss": 0.0116,
|
|
"num_tokens": 53882719.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 912,
|
|
"step_time": 54.70771650224924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1195.0,
|
|
"completions/max_terminated_length": 1195.0,
|
|
"completions/mean_length": 740.140625,
|
|
"completions/mean_terminated_length": 740.140625,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.6221466045826674,
|
|
"epoch": 0.5175736961451247,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.005709858509362675,
|
|
"learning_rate": 4.829931972789115e-07,
|
|
"loss": 0.0232,
|
|
"num_tokens": 53939120.0,
|
|
"reward": 0.5703125,
|
|
"reward_std": 0.8768051266670227,
|
|
"rewards/accuracy_reward_func/mean": 0.53125,
|
|
"rewards/accuracy_reward_func/std": 0.8903138637542725,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 913,
|
|
"step_time": 42.530416775494814
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1221.0,
|
|
"completions/max_terminated_length": 1221.0,
|
|
"completions/mean_length": 786.6875,
|
|
"completions/mean_terminated_length": 786.6875,
|
|
"completions/min_length": 514.0,
|
|
"completions/min_terminated_length": 514.0,
|
|
"entropy": 0.8340959157794714,
|
|
"epoch": 0.518140589569161,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.008421231672400609,
|
|
"learning_rate": 4.824263038548753e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 54002964.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.7949103116989136,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 914,
|
|
"step_time": 44.106495507992804
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1591.0,
|
|
"completions/max_terminated_length": 1591.0,
|
|
"completions/mean_length": 763.703125,
|
|
"completions/mean_terminated_length": 763.703125,
|
|
"completions/min_length": 379.0,
|
|
"completions/min_terminated_length": 379.0,
|
|
"entropy": 0.790501307696104,
|
|
"epoch": 0.5187074829931972,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.007534450574894436,
|
|
"learning_rate": 4.81859410430839e-07,
|
|
"loss": -0.0121,
|
|
"num_tokens": 54062905.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 915,
|
|
"step_time": 56.838639543391764
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1522.0,
|
|
"completions/max_terminated_length": 1522.0,
|
|
"completions/mean_length": 730.234375,
|
|
"completions/mean_terminated_length": 730.234375,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"entropy": 0.8567246794700623,
|
|
"epoch": 0.5192743764172335,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.00953804656455759,
|
|
"learning_rate": 4.812925170068026e-07,
|
|
"loss": 0.0045,
|
|
"num_tokens": 54120568.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 916,
|
|
"step_time": 53.55355898477137
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1292.0,
|
|
"completions/max_terminated_length": 1292.0,
|
|
"completions/mean_length": 717.75,
|
|
"completions/mean_terminated_length": 717.75,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 1.015246793627739,
|
|
"epoch": 0.5198412698412699,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.0063401684747077525,
|
|
"learning_rate": 4.807256235827664e-07,
|
|
"loss": -0.0047,
|
|
"num_tokens": 54181304.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.417855441570282,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 917,
|
|
"step_time": 48.08920597564429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1499.0,
|
|
"completions/max_terminated_length": 1499.0,
|
|
"completions/mean_length": 813.09375,
|
|
"completions/mean_terminated_length": 813.09375,
|
|
"completions/min_length": 449.0,
|
|
"completions/min_terminated_length": 449.0,
|
|
"entropy": 0.7884583976119757,
|
|
"epoch": 0.5204081632653061,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.017351015194435604,
|
|
"learning_rate": 4.801587301587301e-07,
|
|
"loss": 0.0005,
|
|
"num_tokens": 54244814.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.774084210395813,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 918,
|
|
"step_time": 53.87541898339987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1878.0,
|
|
"completions/max_terminated_length": 1878.0,
|
|
"completions/mean_length": 848.59375,
|
|
"completions/mean_terminated_length": 848.59375,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 0.8402353767305613,
|
|
"epoch": 0.5209750566893424,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.006984979336266406,
|
|
"learning_rate": 4.795918367346938e-07,
|
|
"loss": -0.0226,
|
|
"num_tokens": 54308724.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 919,
|
|
"step_time": 65.95115252025425
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1612.0,
|
|
"completions/max_terminated_length": 1612.0,
|
|
"completions/mean_length": 809.171875,
|
|
"completions/mean_terminated_length": 809.171875,
|
|
"completions/min_length": 207.0,
|
|
"completions/min_terminated_length": 207.0,
|
|
"entropy": 0.6449070312082767,
|
|
"epoch": 0.5215419501133787,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.009594561139238067,
|
|
"learning_rate": 4.790249433106575e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 54376847.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 920,
|
|
"step_time": 62.59097481146455
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1992.0,
|
|
"completions/max_terminated_length": 1992.0,
|
|
"completions/mean_length": 686.84375,
|
|
"completions/mean_terminated_length": 686.84375,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.8161975052207708,
|
|
"epoch": 0.5221088435374149,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.00853321272006724,
|
|
"learning_rate": 4.784580498866213e-07,
|
|
"loss": 0.0058,
|
|
"num_tokens": 54430733.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 921,
|
|
"step_time": 70.79646987002343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1405.0,
|
|
"completions/max_terminated_length": 1405.0,
|
|
"completions/mean_length": 797.140625,
|
|
"completions/mean_terminated_length": 797.140625,
|
|
"completions/min_length": 243.0,
|
|
"completions/min_terminated_length": 243.0,
|
|
"entropy": 0.7633548807352781,
|
|
"epoch": 0.5226757369614512,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.005576651499723084,
|
|
"learning_rate": 4.77891156462585e-07,
|
|
"loss": 0.021,
|
|
"num_tokens": 54490606.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 922,
|
|
"step_time": 49.00949365552515
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1471.0,
|
|
"completions/max_terminated_length": 1471.0,
|
|
"completions/mean_length": 768.71875,
|
|
"completions/mean_terminated_length": 768.71875,
|
|
"completions/min_length": 307.0,
|
|
"completions/min_terminated_length": 307.0,
|
|
"entropy": 0.7819051500409842,
|
|
"epoch": 0.5232426303854876,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.01122438551101368,
|
|
"learning_rate": 4.773242630385487e-07,
|
|
"loss": 0.0161,
|
|
"num_tokens": 54551788.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 923,
|
|
"step_time": 52.135914757847786
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1290.0,
|
|
"completions/max_terminated_length": 1290.0,
|
|
"completions/mean_length": 698.21875,
|
|
"completions/mean_terminated_length": 698.21875,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.9958205185830593,
|
|
"epoch": 0.5238095238095238,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4765625,
|
|
"kl": 0.008656778372824192,
|
|
"learning_rate": 4.767573696145125e-07,
|
|
"loss": -0.0295,
|
|
"num_tokens": 54615018.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 924,
|
|
"step_time": 53.114952670410275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1624.0,
|
|
"completions/max_terminated_length": 1624.0,
|
|
"completions/mean_length": 763.296875,
|
|
"completions/mean_terminated_length": 763.296875,
|
|
"completions/min_length": 498.0,
|
|
"completions/min_terminated_length": 498.0,
|
|
"entropy": 0.9542114399373531,
|
|
"epoch": 0.5243764172335601,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.008175756724085659,
|
|
"learning_rate": 4.761904761904761e-07,
|
|
"loss": -0.0025,
|
|
"num_tokens": 54675045.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 925,
|
|
"step_time": 57.15781927481294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1412.0,
|
|
"completions/max_terminated_length": 1412.0,
|
|
"completions/mean_length": 783.859375,
|
|
"completions/mean_terminated_length": 783.859375,
|
|
"completions/min_length": 116.0,
|
|
"completions/min_terminated_length": 116.0,
|
|
"entropy": 0.7769414894282818,
|
|
"epoch": 0.5249433106575964,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.006948598369490355,
|
|
"learning_rate": 4.7562358276643987e-07,
|
|
"loss": 0.0026,
|
|
"num_tokens": 54737724.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 926,
|
|
"step_time": 51.55515736807138
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1597.0,
|
|
"completions/max_terminated_length": 1597.0,
|
|
"completions/mean_length": 760.421875,
|
|
"completions/mean_terminated_length": 760.421875,
|
|
"completions/min_length": 264.0,
|
|
"completions/min_terminated_length": 264.0,
|
|
"entropy": 0.6405221819877625,
|
|
"epoch": 0.5255102040816326,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.00964340116479434,
|
|
"learning_rate": 4.750566893424036e-07,
|
|
"loss": -0.0094,
|
|
"num_tokens": 54796695.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 927,
|
|
"step_time": 55.63045115303248
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1299.0,
|
|
"completions/max_terminated_length": 1299.0,
|
|
"completions/mean_length": 716.625,
|
|
"completions/mean_terminated_length": 716.625,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.8025414180010557,
|
|
"epoch": 0.5260770975056689,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.013545045760110952,
|
|
"learning_rate": 4.7448979591836737e-07,
|
|
"loss": 0.0391,
|
|
"num_tokens": 54852967.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7531349658966064,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 928,
|
|
"step_time": 45.419612736441195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2138.0,
|
|
"completions/max_terminated_length": 2138.0,
|
|
"completions/mean_length": 799.8125,
|
|
"completions/mean_terminated_length": 799.8125,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.5494557935744524,
|
|
"epoch": 0.5266439909297053,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.012848886297433637,
|
|
"learning_rate": 4.73922902494331e-07,
|
|
"loss": 0.0349,
|
|
"num_tokens": 54915163.0,
|
|
"reward": 0.734375,
|
|
"reward_std": 0.9998759627342224,
|
|
"rewards/accuracy_reward_func/mean": 0.6875,
|
|
"rewards/accuracy_reward_func/std": 0.9574271440505981,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 929,
|
|
"step_time": 76.04414348304272
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1402.0,
|
|
"completions/max_terminated_length": 1402.0,
|
|
"completions/mean_length": 767.984375,
|
|
"completions/mean_terminated_length": 767.984375,
|
|
"completions/min_length": 398.0,
|
|
"completions/min_terminated_length": 398.0,
|
|
"entropy": 1.0467991046607494,
|
|
"epoch": 0.5272108843537415,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.018898626876762137,
|
|
"learning_rate": 4.7335600907029476e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 54976762.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 930,
|
|
"step_time": 49.72685145679861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1388.0,
|
|
"completions/max_terminated_length": 1388.0,
|
|
"completions/mean_length": 792.125,
|
|
"completions/mean_terminated_length": 792.125,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.8547965325415134,
|
|
"epoch": 0.5277777777777778,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.259765625,
|
|
"kl": 0.007399451948003843,
|
|
"learning_rate": 4.727891156462585e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 55043282.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 931,
|
|
"step_time": 51.107924149371684
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1562.0,
|
|
"completions/max_terminated_length": 1562.0,
|
|
"completions/mean_length": 740.03125,
|
|
"completions/mean_terminated_length": 740.03125,
|
|
"completions/min_length": 189.0,
|
|
"completions/min_terminated_length": 189.0,
|
|
"entropy": 0.7896527461707592,
|
|
"epoch": 0.528344671201814,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.009829142654780298,
|
|
"learning_rate": 4.722222222222222e-07,
|
|
"loss": -0.0106,
|
|
"num_tokens": 55101700.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5107961297035217,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 932,
|
|
"step_time": 54.63743989728391
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1795.0,
|
|
"completions/max_terminated_length": 1795.0,
|
|
"completions/mean_length": 749.3125,
|
|
"completions/mean_terminated_length": 749.3125,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.6353346854448318,
|
|
"epoch": 0.5289115646258503,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.008822770207189023,
|
|
"learning_rate": 4.716553287981859e-07,
|
|
"loss": 0.0091,
|
|
"num_tokens": 55159448.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 933,
|
|
"step_time": 63.14761694613844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1442.0,
|
|
"completions/max_terminated_length": 1442.0,
|
|
"completions/mean_length": 704.53125,
|
|
"completions/mean_terminated_length": 704.53125,
|
|
"completions/min_length": 246.0,
|
|
"completions/min_terminated_length": 246.0,
|
|
"entropy": 0.885363269597292,
|
|
"epoch": 0.5294784580498866,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.466796875,
|
|
"kl": 0.015152840671362355,
|
|
"learning_rate": 4.7108843537414965e-07,
|
|
"loss": -0.0145,
|
|
"num_tokens": 55216298.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.774684727191925,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 934,
|
|
"step_time": 50.90321496501565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1790.0,
|
|
"completions/max_terminated_length": 1790.0,
|
|
"completions/mean_length": 770.890625,
|
|
"completions/mean_terminated_length": 770.890625,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.7040667347609997,
|
|
"epoch": 0.530045351473923,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.00964798488712404,
|
|
"learning_rate": 4.7052154195011335e-07,
|
|
"loss": 0.0182,
|
|
"num_tokens": 55276027.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 935,
|
|
"step_time": 62.90270910598338
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1245.0,
|
|
"completions/max_terminated_length": 1245.0,
|
|
"completions/mean_length": 789.765625,
|
|
"completions/mean_terminated_length": 789.765625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.8216177001595497,
|
|
"epoch": 0.5306122448979592,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.0072567540482850745,
|
|
"learning_rate": 4.699546485260771e-07,
|
|
"loss": 0.0349,
|
|
"num_tokens": 55337604.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 936,
|
|
"step_time": 44.108981505036354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1258.0,
|
|
"completions/max_terminated_length": 1258.0,
|
|
"completions/mean_length": 717.171875,
|
|
"completions/mean_terminated_length": 717.171875,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.7640928495675325,
|
|
"epoch": 0.5311791383219955,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6796875,
|
|
"kl": 0.0197177168156486,
|
|
"learning_rate": 4.693877551020408e-07,
|
|
"loss": -0.0061,
|
|
"num_tokens": 55392167.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 937,
|
|
"step_time": 44.463840710930526
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1780.0,
|
|
"completions/max_terminated_length": 1780.0,
|
|
"completions/mean_length": 780.421875,
|
|
"completions/mean_terminated_length": 780.421875,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.6826810520142317,
|
|
"epoch": 0.5317460317460317,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.006620996296987869,
|
|
"learning_rate": 4.688208616780045e-07,
|
|
"loss": -0.0131,
|
|
"num_tokens": 55454210.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 938,
|
|
"step_time": 64.50629471056163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1811.0,
|
|
"completions/max_terminated_length": 1811.0,
|
|
"completions/mean_length": 766.390625,
|
|
"completions/mean_terminated_length": 766.390625,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.8801650516688824,
|
|
"epoch": 0.532312925170068,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.00907766679301858,
|
|
"learning_rate": 4.6825396825396824e-07,
|
|
"loss": 0.0533,
|
|
"num_tokens": 55512595.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 939,
|
|
"step_time": 64.18955427594483
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2354.0,
|
|
"completions/max_terminated_length": 2354.0,
|
|
"completions/mean_length": 799.765625,
|
|
"completions/mean_terminated_length": 799.765625,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.874507050961256,
|
|
"epoch": 0.5328798185941043,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.017339513055048883,
|
|
"learning_rate": 4.67687074829932e-07,
|
|
"loss": 0.0159,
|
|
"num_tokens": 55576004.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 940,
|
|
"step_time": 87.24784078169614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1510.0,
|
|
"completions/max_terminated_length": 1510.0,
|
|
"completions/mean_length": 851.390625,
|
|
"completions/mean_terminated_length": 851.390625,
|
|
"completions/min_length": 400.0,
|
|
"completions/min_terminated_length": 400.0,
|
|
"entropy": 0.7663564160466194,
|
|
"epoch": 0.5334467120181405,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.009488253766903654,
|
|
"learning_rate": 4.6712018140589563e-07,
|
|
"loss": 0.0053,
|
|
"num_tokens": 55639357.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7811011672019958,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 941,
|
|
"step_time": 53.032545189373195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1799.0,
|
|
"completions/max_terminated_length": 1799.0,
|
|
"completions/mean_length": 828.203125,
|
|
"completions/mean_terminated_length": 828.203125,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.7306380849331617,
|
|
"epoch": 0.5340136054421769,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.008132042596116662,
|
|
"learning_rate": 4.665532879818594e-07,
|
|
"loss": 0.0359,
|
|
"num_tokens": 55702642.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 942,
|
|
"step_time": 63.776184876449406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1921.0,
|
|
"completions/max_terminated_length": 1921.0,
|
|
"completions/mean_length": 828.734375,
|
|
"completions/mean_terminated_length": 828.734375,
|
|
"completions/min_length": 21.0,
|
|
"completions/min_terminated_length": 21.0,
|
|
"entropy": 0.7034419625997543,
|
|
"epoch": 0.5345804988662132,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.009644535224651918,
|
|
"learning_rate": 4.6598639455782313e-07,
|
|
"loss": 0.0423,
|
|
"num_tokens": 55766745.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 943,
|
|
"step_time": 68.48444921337068
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1718.0,
|
|
"completions/max_terminated_length": 1718.0,
|
|
"completions/mean_length": 828.0625,
|
|
"completions/mean_terminated_length": 828.0625,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"entropy": 0.7754698693752289,
|
|
"epoch": 0.5351473922902494,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.007086289930157363,
|
|
"learning_rate": 4.6541950113378683e-07,
|
|
"loss": 0.0134,
|
|
"num_tokens": 55831997.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 944,
|
|
"step_time": 61.40294525772333
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1156.0,
|
|
"completions/max_terminated_length": 1156.0,
|
|
"completions/mean_length": 726.59375,
|
|
"completions/mean_terminated_length": 726.59375,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.8089466076344252,
|
|
"epoch": 0.5357142857142857,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.012047431082464755,
|
|
"learning_rate": 4.648526077097505e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 55888235.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 945,
|
|
"step_time": 40.44828604161739
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1245.0,
|
|
"completions/max_terminated_length": 1245.0,
|
|
"completions/mean_length": 777.875,
|
|
"completions/mean_terminated_length": 777.875,
|
|
"completions/min_length": 459.0,
|
|
"completions/min_terminated_length": 459.0,
|
|
"entropy": 0.903945554047823,
|
|
"epoch": 0.536281179138322,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"kl": 0.009765000053448603,
|
|
"learning_rate": 4.6428571428571427e-07,
|
|
"loss": 0.0171,
|
|
"num_tokens": 55948683.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 946,
|
|
"step_time": 43.83775487728417
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1405.0,
|
|
"completions/max_terminated_length": 1405.0,
|
|
"completions/mean_length": 795.203125,
|
|
"completions/mean_terminated_length": 795.203125,
|
|
"completions/min_length": 426.0,
|
|
"completions/min_terminated_length": 426.0,
|
|
"entropy": 0.8984325602650642,
|
|
"epoch": 0.5368480725623582,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.007679034548345953,
|
|
"learning_rate": 4.6371882086167797e-07,
|
|
"loss": -0.0153,
|
|
"num_tokens": 56014128.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6281666159629822,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 947,
|
|
"step_time": 50.99723933264613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1828.0,
|
|
"completions/max_terminated_length": 1828.0,
|
|
"completions/mean_length": 761.8125,
|
|
"completions/mean_terminated_length": 761.8125,
|
|
"completions/min_length": 316.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"entropy": 0.8038659747689962,
|
|
"epoch": 0.5374149659863946,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"kl": 0.02042919138330035,
|
|
"learning_rate": 4.631519274376417e-07,
|
|
"loss": 0.0169,
|
|
"num_tokens": 56073908.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 948,
|
|
"step_time": 64.58305048197508
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1188.0,
|
|
"completions/max_terminated_length": 1188.0,
|
|
"completions/mean_length": 735.703125,
|
|
"completions/mean_terminated_length": 735.703125,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 0.9073431864380836,
|
|
"epoch": 0.5379818594104309,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.62890625,
|
|
"kl": 0.011731993785360828,
|
|
"learning_rate": 4.625850340136054e-07,
|
|
"loss": -0.0007,
|
|
"num_tokens": 56132545.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7478474378585815,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 949,
|
|
"step_time": 42.67391610145569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2200.0,
|
|
"completions/max_terminated_length": 2200.0,
|
|
"completions/mean_length": 783.28125,
|
|
"completions/mean_terminated_length": 783.28125,
|
|
"completions/min_length": 231.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"entropy": 1.0546772629022598,
|
|
"epoch": 0.5385487528344671,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.00719371362356469,
|
|
"learning_rate": 4.620181405895691e-07,
|
|
"loss": -0.0171,
|
|
"num_tokens": 56192963.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 950,
|
|
"step_time": 78.57215535920113
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1468.0,
|
|
"completions/max_terminated_length": 1468.0,
|
|
"completions/mean_length": 900.078125,
|
|
"completions/mean_terminated_length": 900.078125,
|
|
"completions/min_length": 443.0,
|
|
"completions/min_terminated_length": 443.0,
|
|
"entropy": 0.7257055677473545,
|
|
"epoch": 0.5391156462585034,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.3984375,
|
|
"kl": 0.007188014467828907,
|
|
"learning_rate": 4.6145124716553286e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 56261944.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 951,
|
|
"step_time": 52.33880191762
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1448.0,
|
|
"completions/max_terminated_length": 1448.0,
|
|
"completions/mean_length": 800.640625,
|
|
"completions/mean_terminated_length": 800.640625,
|
|
"completions/min_length": 361.0,
|
|
"completions/min_terminated_length": 361.0,
|
|
"entropy": 0.720583526417613,
|
|
"epoch": 0.5396825396825397,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.006856111242086627,
|
|
"learning_rate": 4.608843537414966e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 56323913.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 952,
|
|
"step_time": 51.751177513971925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1290.0,
|
|
"completions/max_terminated_length": 1290.0,
|
|
"completions/mean_length": 722.765625,
|
|
"completions/mean_terminated_length": 722.765625,
|
|
"completions/min_length": 163.0,
|
|
"completions/min_terminated_length": 163.0,
|
|
"entropy": 0.7319106739014387,
|
|
"epoch": 0.5402494331065759,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.009742267386172898,
|
|
"learning_rate": 4.6031746031746025e-07,
|
|
"loss": 0.024,
|
|
"num_tokens": 56380242.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 953,
|
|
"step_time": 45.43985826894641
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1801.0,
|
|
"completions/max_terminated_length": 1801.0,
|
|
"completions/mean_length": 919.0625,
|
|
"completions/mean_terminated_length": 919.0625,
|
|
"completions/min_length": 477.0,
|
|
"completions/min_terminated_length": 477.0,
|
|
"entropy": 0.6944993417710066,
|
|
"epoch": 0.5408163265306123,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.006739786796970293,
|
|
"learning_rate": 4.59750566893424e-07,
|
|
"loss": -0.017,
|
|
"num_tokens": 56449774.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 954,
|
|
"step_time": 63.771343723870814
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2484.0,
|
|
"completions/max_terminated_length": 2484.0,
|
|
"completions/mean_length": 825.890625,
|
|
"completions/mean_terminated_length": 825.890625,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.8774555139243603,
|
|
"epoch": 0.5413832199546486,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.400390625,
|
|
"kl": 0.00560982410388533,
|
|
"learning_rate": 4.5918367346938775e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 56512295.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.556738555431366,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 955,
|
|
"step_time": 90.87684091739357
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1149.0,
|
|
"completions/max_terminated_length": 1149.0,
|
|
"completions/mean_length": 695.59375,
|
|
"completions/mean_terminated_length": 695.59375,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.8045432157814503,
|
|
"epoch": 0.5419501133786848,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.009588926623109728,
|
|
"learning_rate": 4.5861678004535145e-07,
|
|
"loss": -0.0155,
|
|
"num_tokens": 56570277.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 956,
|
|
"step_time": 41.767555746249855
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1295.0,
|
|
"completions/max_terminated_length": 1295.0,
|
|
"completions/mean_length": 696.953125,
|
|
"completions/mean_terminated_length": 696.953125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 0.59090162999928,
|
|
"epoch": 0.5425170068027211,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.005754989237175323,
|
|
"learning_rate": 4.5804988662131514e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 56624050.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.7753247618675232,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 957,
|
|
"step_time": 45.85981133300811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1052.0,
|
|
"completions/max_terminated_length": 1052.0,
|
|
"completions/mean_length": 688.609375,
|
|
"completions/mean_terminated_length": 688.609375,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 1.0952195599675179,
|
|
"epoch": 0.5430839002267573,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.7109375,
|
|
"kl": 0.011699286274961196,
|
|
"learning_rate": 4.574829931972789e-07,
|
|
"loss": 0.0099,
|
|
"num_tokens": 56679801.0,
|
|
"reward": 0.484375,
|
|
"reward_std": 0.8211913108825684,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 958,
|
|
"step_time": 37.894682499580085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1303.0,
|
|
"completions/max_terminated_length": 1303.0,
|
|
"completions/mean_length": 763.6875,
|
|
"completions/mean_terminated_length": 763.6875,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 0.9255213402211666,
|
|
"epoch": 0.5436507936507936,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.008241173665737733,
|
|
"learning_rate": 4.5691609977324264e-07,
|
|
"loss": -0.01,
|
|
"num_tokens": 56740213.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 959,
|
|
"step_time": 46.291500895284116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1320.0,
|
|
"completions/max_terminated_length": 1320.0,
|
|
"completions/mean_length": 677.484375,
|
|
"completions/mean_terminated_length": 677.484375,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.9439113810658455,
|
|
"epoch": 0.54421768707483,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.006982455437537283,
|
|
"learning_rate": 4.5634920634920634e-07,
|
|
"loss": 0.0331,
|
|
"num_tokens": 56793764.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 960,
|
|
"step_time": 46.60715462453663
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1407.0,
|
|
"completions/max_terminated_length": 1407.0,
|
|
"completions/mean_length": 648.40625,
|
|
"completions/mean_terminated_length": 648.40625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.603379238396883,
|
|
"epoch": 0.5447845804988662,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.013042758815572597,
|
|
"learning_rate": 4.5578231292517003e-07,
|
|
"loss": 0.0261,
|
|
"num_tokens": 56846542.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.7943251132965088,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 961,
|
|
"step_time": 50.20637048408389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1122.0,
|
|
"completions/max_terminated_length": 1122.0,
|
|
"completions/mean_length": 656.296875,
|
|
"completions/mean_terminated_length": 656.296875,
|
|
"completions/min_length": 234.0,
|
|
"completions/min_terminated_length": 234.0,
|
|
"entropy": 0.8278116472065449,
|
|
"epoch": 0.5453514739229025,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.60546875,
|
|
"kl": 0.015698245872044936,
|
|
"learning_rate": 4.552154195011338e-07,
|
|
"loss": 0.0374,
|
|
"num_tokens": 56907537.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 962,
|
|
"step_time": 45.608940177597106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1817.0,
|
|
"completions/max_terminated_length": 1817.0,
|
|
"completions/mean_length": 782.390625,
|
|
"completions/mean_terminated_length": 782.390625,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.7491417396813631,
|
|
"epoch": 0.5459183673469388,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.006047677205060609,
|
|
"learning_rate": 4.546485260770975e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 56971258.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.7545743584632874,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 963,
|
|
"step_time": 65.96868342068046
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2489.0,
|
|
"completions/max_terminated_length": 2489.0,
|
|
"completions/mean_length": 810.96875,
|
|
"completions/mean_terminated_length": 810.96875,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.7321901936084032,
|
|
"epoch": 0.546485260770975,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.009162333051790483,
|
|
"learning_rate": 4.5408163265306123e-07,
|
|
"loss": 0.0256,
|
|
"num_tokens": 57033784.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.665922224521637,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 964,
|
|
"step_time": 91.77159368712455
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1954.0,
|
|
"completions/max_terminated_length": 1954.0,
|
|
"completions/mean_length": 860.296875,
|
|
"completions/mean_terminated_length": 860.296875,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.8021142072975636,
|
|
"epoch": 0.5470521541950113,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.006959985112189315,
|
|
"learning_rate": 4.535147392290249e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 57102635.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 965,
|
|
"step_time": 71.01779305934906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1492.0,
|
|
"completions/max_terminated_length": 1492.0,
|
|
"completions/mean_length": 716.125,
|
|
"completions/mean_terminated_length": 716.125,
|
|
"completions/min_length": 193.0,
|
|
"completions/min_terminated_length": 193.0,
|
|
"entropy": 0.822033978998661,
|
|
"epoch": 0.5476190476190477,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.0074520827329251915,
|
|
"learning_rate": 4.529478458049886e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 57158259.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 966,
|
|
"step_time": 52.46762257814407
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1439.0,
|
|
"completions/max_terminated_length": 1439.0,
|
|
"completions/mean_length": 706.546875,
|
|
"completions/mean_terminated_length": 706.546875,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.9053959883749485,
|
|
"epoch": 0.5481859410430839,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.01610620430437848,
|
|
"learning_rate": 4.5238095238095237e-07,
|
|
"loss": -0.0131,
|
|
"num_tokens": 57213806.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 967,
|
|
"step_time": 50.41593927517533
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1289.0,
|
|
"completions/max_terminated_length": 1289.0,
|
|
"completions/mean_length": 670.6875,
|
|
"completions/mean_terminated_length": 670.6875,
|
|
"completions/min_length": 171.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"entropy": 0.8066293485462666,
|
|
"epoch": 0.5487528344671202,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.008681508246809244,
|
|
"learning_rate": 4.518140589569161e-07,
|
|
"loss": -0.018,
|
|
"num_tokens": 57266330.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 968,
|
|
"step_time": 44.949662261642516
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1439.0,
|
|
"completions/max_terminated_length": 1439.0,
|
|
"completions/mean_length": 825.84375,
|
|
"completions/mean_terminated_length": 825.84375,
|
|
"completions/min_length": 360.0,
|
|
"completions/min_terminated_length": 360.0,
|
|
"entropy": 0.7035286258906126,
|
|
"epoch": 0.5493197278911565,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.011340483397361822,
|
|
"learning_rate": 4.5124716553287976e-07,
|
|
"loss": -0.0124,
|
|
"num_tokens": 57331344.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 969,
|
|
"step_time": 51.598348090425134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1097.0,
|
|
"completions/max_terminated_length": 1097.0,
|
|
"completions/mean_length": 684.796875,
|
|
"completions/mean_terminated_length": 684.796875,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 1.177319511771202,
|
|
"epoch": 0.5498866213151927,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.009384535980643705,
|
|
"learning_rate": 4.506802721088435e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 57386075.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 970,
|
|
"step_time": 42.747706608846784
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2043.0,
|
|
"completions/max_terminated_length": 2043.0,
|
|
"completions/mean_length": 750.609375,
|
|
"completions/mean_terminated_length": 750.609375,
|
|
"completions/min_length": 154.0,
|
|
"completions/min_terminated_length": 154.0,
|
|
"entropy": 0.6261631045490503,
|
|
"epoch": 0.550453514739229,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.00568970781750977,
|
|
"learning_rate": 4.5011337868480726e-07,
|
|
"loss": -0.001,
|
|
"num_tokens": 57442626.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 971,
|
|
"step_time": 72.4735603723675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1163.0,
|
|
"completions/max_terminated_length": 1163.0,
|
|
"completions/mean_length": 714.140625,
|
|
"completions/mean_terminated_length": 714.140625,
|
|
"completions/min_length": 190.0,
|
|
"completions/min_terminated_length": 190.0,
|
|
"entropy": 0.7821238599717617,
|
|
"epoch": 0.5510204081632653,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.008154489667504095,
|
|
"learning_rate": 4.4954648526077096e-07,
|
|
"loss": 0.0305,
|
|
"num_tokens": 57498571.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 972,
|
|
"step_time": 41.223891040310264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1398.0,
|
|
"completions/max_terminated_length": 1398.0,
|
|
"completions/mean_length": 744.109375,
|
|
"completions/mean_terminated_length": 744.109375,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.8839225824922323,
|
|
"epoch": 0.5515873015873016,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.02203242588439025,
|
|
"learning_rate": 4.4897959183673465e-07,
|
|
"loss": -0.0078,
|
|
"num_tokens": 57558394.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.7545743584632874,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 973,
|
|
"step_time": 49.371847248636186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1997.0,
|
|
"completions/max_terminated_length": 1997.0,
|
|
"completions/mean_length": 742.546875,
|
|
"completions/mean_terminated_length": 742.546875,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.7721455320715904,
|
|
"epoch": 0.5521541950113379,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.009138341876678169,
|
|
"learning_rate": 4.484126984126984e-07,
|
|
"loss": -0.0074,
|
|
"num_tokens": 57615429.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 974,
|
|
"step_time": 70.90645644720644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1332.0,
|
|
"completions/max_terminated_length": 1332.0,
|
|
"completions/mean_length": 735.9375,
|
|
"completions/mean_terminated_length": 735.9375,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.892586562782526,
|
|
"epoch": 0.5527210884353742,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.010881248133955523,
|
|
"learning_rate": 4.478458049886621e-07,
|
|
"loss": 0.021,
|
|
"num_tokens": 57673673.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 975,
|
|
"step_time": 47.053733562119305
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1423.0,
|
|
"completions/max_terminated_length": 1423.0,
|
|
"completions/mean_length": 819.3125,
|
|
"completions/mean_terminated_length": 819.3125,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.7769006192684174,
|
|
"epoch": 0.5532879818594104,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.014836488859145902,
|
|
"learning_rate": 4.4727891156462585e-07,
|
|
"loss": -0.0106,
|
|
"num_tokens": 57738301.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.8020543456077576,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 976,
|
|
"step_time": 51.44586255867034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1675.0,
|
|
"completions/max_terminated_length": 1675.0,
|
|
"completions/mean_length": 663.953125,
|
|
"completions/mean_terminated_length": 663.953125,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 0.6291102189570665,
|
|
"epoch": 0.5538548752834467,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"kl": 0.011376376249245368,
|
|
"learning_rate": 4.4671201814058955e-07,
|
|
"loss": -0.0138,
|
|
"num_tokens": 57790090.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 977,
|
|
"step_time": 59.21410261001438
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1234.0,
|
|
"completions/max_terminated_length": 1234.0,
|
|
"completions/mean_length": 645.640625,
|
|
"completions/mean_terminated_length": 645.640625,
|
|
"completions/min_length": 181.0,
|
|
"completions/min_terminated_length": 181.0,
|
|
"entropy": 0.7397663220763206,
|
|
"epoch": 0.5544217687074829,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.013661038246937096,
|
|
"learning_rate": 4.4614512471655324e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 57842003.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.8207381367683411,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 978,
|
|
"step_time": 44.70080322306603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1488.0,
|
|
"completions/max_terminated_length": 1488.0,
|
|
"completions/mean_length": 724.765625,
|
|
"completions/mean_terminated_length": 724.765625,
|
|
"completions/min_length": 324.0,
|
|
"completions/min_terminated_length": 324.0,
|
|
"entropy": 0.9115379229187965,
|
|
"epoch": 0.5549886621315193,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.008456474606646225,
|
|
"learning_rate": 4.45578231292517e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 57899916.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 979,
|
|
"step_time": 52.325842208229005
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1246.0,
|
|
"completions/max_terminated_length": 1246.0,
|
|
"completions/mean_length": 764.5,
|
|
"completions/mean_terminated_length": 764.5,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.7857040986418724,
|
|
"epoch": 0.5555555555555556,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.890625,
|
|
"kl": 0.008615031081717461,
|
|
"learning_rate": 4.4501133786848074e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 57957708.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.6574888825416565,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.109375,
|
|
"rewards/format_reward_func/std": 0.2083333432674408,
|
|
"step": 980,
|
|
"step_time": 43.65809071995318
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1678.0,
|
|
"completions/max_terminated_length": 1678.0,
|
|
"completions/mean_length": 830.140625,
|
|
"completions/mean_terminated_length": 830.140625,
|
|
"completions/min_length": 390.0,
|
|
"completions/min_terminated_length": 390.0,
|
|
"entropy": 0.8238047361373901,
|
|
"epoch": 0.5561224489795918,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.01065999582351651,
|
|
"learning_rate": 4.444444444444444e-07,
|
|
"loss": 0.0222,
|
|
"num_tokens": 58022053.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 981,
|
|
"step_time": 59.01628066599369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1765.0,
|
|
"completions/max_terminated_length": 1765.0,
|
|
"completions/mean_length": 724.234375,
|
|
"completions/mean_terminated_length": 724.234375,
|
|
"completions/min_length": 213.0,
|
|
"completions/min_terminated_length": 213.0,
|
|
"entropy": 0.8812555447220802,
|
|
"epoch": 0.5566893424036281,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.010285356896929443,
|
|
"learning_rate": 4.4387755102040813e-07,
|
|
"loss": -0.0039,
|
|
"num_tokens": 58087508.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 982,
|
|
"step_time": 72.91728581395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1665.0,
|
|
"completions/max_terminated_length": 1665.0,
|
|
"completions/mean_length": 798.328125,
|
|
"completions/mean_terminated_length": 798.328125,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.7892794460058212,
|
|
"epoch": 0.5572562358276644,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.009645198864745907,
|
|
"learning_rate": 4.433106575963719e-07,
|
|
"loss": -0.0036,
|
|
"num_tokens": 58151297.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.528690755367279,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 983,
|
|
"step_time": 59.699283415451646
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1545.0,
|
|
"completions/max_terminated_length": 1545.0,
|
|
"completions/mean_length": 759.9375,
|
|
"completions/mean_terminated_length": 759.9375,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.735388346016407,
|
|
"epoch": 0.5578231292517006,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.006335705533274449,
|
|
"learning_rate": 4.427437641723356e-07,
|
|
"loss": 0.0222,
|
|
"num_tokens": 58217797.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 984,
|
|
"step_time": 61.201052052900195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1828.0,
|
|
"completions/max_terminated_length": 1828.0,
|
|
"completions/mean_length": 758.546875,
|
|
"completions/mean_terminated_length": 758.546875,
|
|
"completions/min_length": 219.0,
|
|
"completions/min_terminated_length": 219.0,
|
|
"entropy": 0.7845967765897512,
|
|
"epoch": 0.558390022675737,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.009897444368107244,
|
|
"learning_rate": 4.421768707482993e-07,
|
|
"loss": -0.0059,
|
|
"num_tokens": 58277632.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.7116082906723022,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 985,
|
|
"step_time": 65.77875246480107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1364.0,
|
|
"completions/max_terminated_length": 1364.0,
|
|
"completions/mean_length": 778.3125,
|
|
"completions/mean_terminated_length": 778.3125,
|
|
"completions/min_length": 173.0,
|
|
"completions/min_terminated_length": 173.0,
|
|
"entropy": 0.8510223850607872,
|
|
"epoch": 0.5589569160997733,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.012109777744626626,
|
|
"learning_rate": 4.41609977324263e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 58338660.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 986,
|
|
"step_time": 48.748002890497446
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1219.0,
|
|
"completions/max_terminated_length": 1219.0,
|
|
"completions/mean_length": 702.78125,
|
|
"completions/mean_terminated_length": 702.78125,
|
|
"completions/min_length": 256.0,
|
|
"completions/min_terminated_length": 256.0,
|
|
"entropy": 0.9813946858048439,
|
|
"epoch": 0.5595238095238095,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.412109375,
|
|
"kl": 0.015161136092501692,
|
|
"learning_rate": 4.4104308390022677e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 58397302.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 987,
|
|
"step_time": 44.04843788500875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1230.0,
|
|
"completions/mean_length": 770.96875,
|
|
"completions/mean_terminated_length": 718.1904907226562,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.7030325084924698,
|
|
"epoch": 0.5600907029478458,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.359375,
|
|
"kl": 0.005897752693272196,
|
|
"learning_rate": 4.4047619047619047e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 58459180.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 988,
|
|
"step_time": 177.64333445206285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1417.0,
|
|
"completions/max_terminated_length": 1417.0,
|
|
"completions/mean_length": 757.296875,
|
|
"completions/mean_terminated_length": 757.296875,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.7383997142314911,
|
|
"epoch": 0.5606575963718821,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"kl": 0.009333990892628208,
|
|
"learning_rate": 4.3990929705215417e-07,
|
|
"loss": -0.0014,
|
|
"num_tokens": 58517791.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 989,
|
|
"step_time": 49.946804178878665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1219.0,
|
|
"completions/max_terminated_length": 1219.0,
|
|
"completions/mean_length": 657.359375,
|
|
"completions/mean_terminated_length": 657.359375,
|
|
"completions/min_length": 274.0,
|
|
"completions/min_terminated_length": 274.0,
|
|
"entropy": 0.916077870875597,
|
|
"epoch": 0.5612244897959183,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58984375,
|
|
"kl": 0.012768377433530986,
|
|
"learning_rate": 4.393424036281179e-07,
|
|
"loss": 0.0293,
|
|
"num_tokens": 58573382.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7478474378585815,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 990,
|
|
"step_time": 45.574856635183096
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1435.0,
|
|
"completions/max_terminated_length": 1435.0,
|
|
"completions/mean_length": 776.84375,
|
|
"completions/mean_terminated_length": 776.84375,
|
|
"completions/min_length": 433.0,
|
|
"completions/min_terminated_length": 433.0,
|
|
"entropy": 0.7702713683247566,
|
|
"epoch": 0.5617913832199547,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.490234375,
|
|
"kl": 0.006071896743378602,
|
|
"learning_rate": 4.387755102040816e-07,
|
|
"loss": -0.0127,
|
|
"num_tokens": 58632820.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.4525473415851593,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.1171875,
|
|
"rewards/format_reward_func/std": 0.21347814798355103,
|
|
"step": 991,
|
|
"step_time": 50.66554786358029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1494.0,
|
|
"completions/max_terminated_length": 1494.0,
|
|
"completions/mean_length": 892.5625,
|
|
"completions/mean_terminated_length": 892.5625,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.7653214447200298,
|
|
"epoch": 0.562358276643991,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.0068724896118510514,
|
|
"learning_rate": 4.3820861678004536e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 58700520.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 992,
|
|
"step_time": 53.4002155167982
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1429.0,
|
|
"completions/max_terminated_length": 1429.0,
|
|
"completions/mean_length": 809.734375,
|
|
"completions/mean_terminated_length": 809.734375,
|
|
"completions/min_length": 497.0,
|
|
"completions/min_terminated_length": 497.0,
|
|
"entropy": 0.7048439346253872,
|
|
"epoch": 0.5629251700680272,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.007161693516536616,
|
|
"learning_rate": 4.3764172335600906e-07,
|
|
"loss": 0.0017,
|
|
"num_tokens": 58763175.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 993,
|
|
"step_time": 50.57035450078547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1200.0,
|
|
"completions/max_terminated_length": 1200.0,
|
|
"completions/mean_length": 703.78125,
|
|
"completions/mean_terminated_length": 703.78125,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.9395108409225941,
|
|
"epoch": 0.5634920634920635,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.011840565770398825,
|
|
"learning_rate": 4.3707482993197275e-07,
|
|
"loss": 0.0083,
|
|
"num_tokens": 58819809.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 994,
|
|
"step_time": 42.9371617520228
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1357.0,
|
|
"completions/max_terminated_length": 1357.0,
|
|
"completions/mean_length": 646.8125,
|
|
"completions/mean_terminated_length": 646.8125,
|
|
"completions/min_length": 219.0,
|
|
"completions/min_terminated_length": 219.0,
|
|
"entropy": 0.8610539846122265,
|
|
"epoch": 0.5640589569160998,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.78125,
|
|
"kl": 0.04494221956701949,
|
|
"learning_rate": 4.365079365079365e-07,
|
|
"loss": -0.0396,
|
|
"num_tokens": 58872253.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7911965847015381,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 995,
|
|
"step_time": 48.01166058611125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1405.0,
|
|
"completions/max_terminated_length": 1405.0,
|
|
"completions/mean_length": 711.34375,
|
|
"completions/mean_terminated_length": 711.34375,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 0.9638322554528713,
|
|
"epoch": 0.564625850340136,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.70703125,
|
|
"kl": 0.0071011144609656185,
|
|
"learning_rate": 4.3594104308390025e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 58929451.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 996,
|
|
"step_time": 50.24006215389818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1756.0,
|
|
"completions/max_terminated_length": 1756.0,
|
|
"completions/mean_length": 772.0625,
|
|
"completions/mean_terminated_length": 772.0625,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.8995952680706978,
|
|
"epoch": 0.5651927437641724,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"kl": 0.00719841496902518,
|
|
"learning_rate": 4.353741496598639e-07,
|
|
"loss": 0.0026,
|
|
"num_tokens": 58992303.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 997,
|
|
"step_time": 63.56839912943542
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1275.0,
|
|
"completions/max_terminated_length": 1275.0,
|
|
"completions/mean_length": 728.75,
|
|
"completions/mean_terminated_length": 728.75,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.5709743648767471,
|
|
"epoch": 0.5657596371882087,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4375,
|
|
"kl": 0.018575849826447666,
|
|
"learning_rate": 4.3480725623582764e-07,
|
|
"loss": 0.0077,
|
|
"num_tokens": 59050479.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 998,
|
|
"step_time": 45.58077103924006
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1931.0,
|
|
"completions/max_terminated_length": 1931.0,
|
|
"completions/mean_length": 912.859375,
|
|
"completions/mean_terminated_length": 912.859375,
|
|
"completions/min_length": 404.0,
|
|
"completions/min_terminated_length": 404.0,
|
|
"entropy": 0.6420471537858248,
|
|
"epoch": 0.5663265306122449,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.392578125,
|
|
"kl": 0.005370252154534683,
|
|
"learning_rate": 4.342403628117914e-07,
|
|
"loss": -0.0257,
|
|
"num_tokens": 59118750.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.559903621673584,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 999,
|
|
"step_time": 69.28126693982631
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1082.0,
|
|
"completions/max_terminated_length": 1082.0,
|
|
"completions/mean_length": 673.734375,
|
|
"completions/mean_terminated_length": 673.734375,
|
|
"completions/min_length": 247.0,
|
|
"completions/min_terminated_length": 247.0,
|
|
"entropy": 0.8986706025898457,
|
|
"epoch": 0.5668934240362812,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.62109375,
|
|
"kl": 0.007250033537275158,
|
|
"learning_rate": 4.336734693877551e-07,
|
|
"loss": -0.0207,
|
|
"num_tokens": 59173525.0,
|
|
"reward": 0.5234375,
|
|
"reward_std": 0.8564863801002502,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1000,
|
|
"step_time": 39.07200087606907
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1469.0,
|
|
"completions/max_terminated_length": 1469.0,
|
|
"completions/mean_length": 693.15625,
|
|
"completions/mean_terminated_length": 693.15625,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 1.0774033926427364,
|
|
"epoch": 0.5674603174603174,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.016356758031179197,
|
|
"learning_rate": 4.331065759637188e-07,
|
|
"loss": -0.0067,
|
|
"num_tokens": 59233735.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1001,
|
|
"step_time": 56.94277353025973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1358.0,
|
|
"completions/max_terminated_length": 1358.0,
|
|
"completions/mean_length": 733.78125,
|
|
"completions/mean_terminated_length": 733.78125,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.7500897962599993,
|
|
"epoch": 0.5680272108843537,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3203125,
|
|
"kl": 0.008228347491240129,
|
|
"learning_rate": 4.3253968253968253e-07,
|
|
"loss": -0.0051,
|
|
"num_tokens": 59291041.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1002,
|
|
"step_time": 48.464056487195194
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1428.0,
|
|
"completions/max_terminated_length": 1428.0,
|
|
"completions/mean_length": 784.921875,
|
|
"completions/mean_terminated_length": 784.921875,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.8411187641322613,
|
|
"epoch": 0.56859410430839,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.396484375,
|
|
"kl": 0.008222160424338654,
|
|
"learning_rate": 4.3197278911564623e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 59352452.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1003,
|
|
"step_time": 50.64359785988927
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1113.0,
|
|
"completions/max_terminated_length": 1113.0,
|
|
"completions/mean_length": 651.96875,
|
|
"completions/mean_terminated_length": 651.96875,
|
|
"completions/min_length": 206.0,
|
|
"completions/min_terminated_length": 206.0,
|
|
"entropy": 0.9559594839811325,
|
|
"epoch": 0.5691609977324263,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.35546875,
|
|
"kl": 0.00953879150620196,
|
|
"learning_rate": 4.3140589569161e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 59406298.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1004,
|
|
"step_time": 40.20570925436914
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1412.0,
|
|
"completions/max_terminated_length": 1412.0,
|
|
"completions/mean_length": 710.984375,
|
|
"completions/mean_terminated_length": 710.984375,
|
|
"completions/min_length": 209.0,
|
|
"completions/min_terminated_length": 209.0,
|
|
"entropy": 0.8136644046753645,
|
|
"epoch": 0.5697278911564626,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"kl": 0.010079790401505306,
|
|
"learning_rate": 4.308390022675737e-07,
|
|
"loss": -0.025,
|
|
"num_tokens": 59462161.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1005,
|
|
"step_time": 49.644781799055636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1665.0,
|
|
"completions/max_terminated_length": 1665.0,
|
|
"completions/mean_length": 856.671875,
|
|
"completions/mean_terminated_length": 856.671875,
|
|
"completions/min_length": 421.0,
|
|
"completions/min_terminated_length": 421.0,
|
|
"entropy": 0.770473551005125,
|
|
"epoch": 0.5702947845804989,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.369140625,
|
|
"kl": 0.0081905969127547,
|
|
"learning_rate": 4.3027210884353737e-07,
|
|
"loss": -0.0123,
|
|
"num_tokens": 59528884.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7614454030990601,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1006,
|
|
"step_time": 59.003919675946236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1620.0,
|
|
"completions/max_terminated_length": 1620.0,
|
|
"completions/mean_length": 787.078125,
|
|
"completions/mean_terminated_length": 787.078125,
|
|
"completions/min_length": 432.0,
|
|
"completions/min_terminated_length": 432.0,
|
|
"entropy": 1.0377478003501892,
|
|
"epoch": 0.5708616780045351,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.008805297038634308,
|
|
"learning_rate": 4.297052154195011e-07,
|
|
"loss": 0.0093,
|
|
"num_tokens": 59589769.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1007,
|
|
"step_time": 57.76021850388497
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1644.0,
|
|
"completions/max_terminated_length": 1644.0,
|
|
"completions/mean_length": 771.4375,
|
|
"completions/mean_terminated_length": 771.4375,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.7078898372128606,
|
|
"epoch": 0.5714285714285714,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.006918784158187918,
|
|
"learning_rate": 4.2913832199546487e-07,
|
|
"loss": -0.0104,
|
|
"num_tokens": 59649437.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7230611443519592,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1008,
|
|
"step_time": 58.71829978656024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1510.0,
|
|
"completions/max_terminated_length": 1510.0,
|
|
"completions/mean_length": 597.609375,
|
|
"completions/mean_terminated_length": 597.609375,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 1.0089727453887463,
|
|
"epoch": 0.5719954648526077,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.62890625,
|
|
"kl": 0.01330042896734085,
|
|
"learning_rate": 4.285714285714285e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 59699100.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1009,
|
|
"step_time": 53.67040815204382
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1321.0,
|
|
"completions/max_terminated_length": 1321.0,
|
|
"completions/mean_length": 752.609375,
|
|
"completions/mean_terminated_length": 752.609375,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.6274698805063963,
|
|
"epoch": 0.572562358276644,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6171875,
|
|
"kl": 0.0057345866080140695,
|
|
"learning_rate": 4.2800453514739226e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 59755939.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7516103982925415,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1010,
|
|
"step_time": 46.80544613208622
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/max_terminated_length": 1376.0,
|
|
"completions/mean_length": 797.953125,
|
|
"completions/mean_terminated_length": 797.953125,
|
|
"completions/min_length": 428.0,
|
|
"completions/min_terminated_length": 428.0,
|
|
"entropy": 0.8501798808574677,
|
|
"epoch": 0.5731292517006803,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25390625,
|
|
"kl": 0.009817527010454796,
|
|
"learning_rate": 4.27437641723356e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 59817512.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1011,
|
|
"step_time": 48.41082198731601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1706.0,
|
|
"completions/max_terminated_length": 1706.0,
|
|
"completions/mean_length": 827.203125,
|
|
"completions/mean_terminated_length": 827.203125,
|
|
"completions/min_length": 378.0,
|
|
"completions/min_terminated_length": 378.0,
|
|
"entropy": 0.644038550555706,
|
|
"epoch": 0.5736961451247166,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.296875,
|
|
"kl": 0.01552946979063563,
|
|
"learning_rate": 4.268707482993197e-07,
|
|
"loss": -0.0021,
|
|
"num_tokens": 59883701.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1012,
|
|
"step_time": 61.67431230377406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1559.0,
|
|
"completions/max_terminated_length": 1559.0,
|
|
"completions/mean_length": 823.140625,
|
|
"completions/mean_terminated_length": 823.140625,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.6703172046691179,
|
|
"epoch": 0.5742630385487528,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.008637107152026147,
|
|
"learning_rate": 4.263038548752834e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 59947430.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1013,
|
|
"step_time": 55.66849758196622
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1333.0,
|
|
"completions/max_terminated_length": 1333.0,
|
|
"completions/mean_length": 701.234375,
|
|
"completions/mean_terminated_length": 701.234375,
|
|
"completions/min_length": 322.0,
|
|
"completions/min_terminated_length": 322.0,
|
|
"entropy": 0.8897392228245735,
|
|
"epoch": 0.5748299319727891,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.008595576044172049,
|
|
"learning_rate": 4.2573696145124715e-07,
|
|
"loss": -0.0097,
|
|
"num_tokens": 60003589.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1014,
|
|
"step_time": 47.25219943281263
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1322.0,
|
|
"completions/max_terminated_length": 1322.0,
|
|
"completions/mean_length": 780.28125,
|
|
"completions/mean_terminated_length": 780.28125,
|
|
"completions/min_length": 21.0,
|
|
"completions/min_terminated_length": 21.0,
|
|
"entropy": 0.6642551589757204,
|
|
"epoch": 0.5753968253968254,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.00779375342244748,
|
|
"learning_rate": 4.2517006802721085e-07,
|
|
"loss": 0.0014,
|
|
"num_tokens": 60064623.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1015,
|
|
"step_time": 46.80390986986458
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1155.0,
|
|
"completions/max_terminated_length": 1155.0,
|
|
"completions/mean_length": 728.15625,
|
|
"completions/mean_terminated_length": 728.15625,
|
|
"completions/min_length": 476.0,
|
|
"completions/min_terminated_length": 476.0,
|
|
"entropy": 1.0858629047870636,
|
|
"epoch": 0.5759637188208617,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.015303319290978834,
|
|
"learning_rate": 4.246031746031746e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 60120809.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1016,
|
|
"step_time": 40.74859708361328
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1725.0,
|
|
"completions/max_terminated_length": 1725.0,
|
|
"completions/mean_length": 862.515625,
|
|
"completions/mean_terminated_length": 862.515625,
|
|
"completions/min_length": 306.0,
|
|
"completions/min_terminated_length": 306.0,
|
|
"entropy": 0.6339191198348999,
|
|
"epoch": 0.576530612244898,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.008789016079390422,
|
|
"learning_rate": 4.240362811791383e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 60188050.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 1017,
|
|
"step_time": 60.94161330163479
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1190.0,
|
|
"completions/max_terminated_length": 1190.0,
|
|
"completions/mean_length": 712.0,
|
|
"completions/mean_terminated_length": 712.0,
|
|
"completions/min_length": 212.0,
|
|
"completions/min_terminated_length": 212.0,
|
|
"entropy": 0.8736541010439396,
|
|
"epoch": 0.5770975056689343,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.008748070627916604,
|
|
"learning_rate": 4.2346938775510205e-07,
|
|
"loss": -0.0022,
|
|
"num_tokens": 60243138.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1018,
|
|
"step_time": 42.00453090853989
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1245.0,
|
|
"completions/max_terminated_length": 1245.0,
|
|
"completions/mean_length": 674.34375,
|
|
"completions/mean_terminated_length": 674.34375,
|
|
"completions/min_length": 224.0,
|
|
"completions/min_terminated_length": 224.0,
|
|
"entropy": 0.939565397799015,
|
|
"epoch": 0.5776643990929705,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6328125,
|
|
"kl": 0.008843418472679332,
|
|
"learning_rate": 4.2290249433106574e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 60296544.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1019,
|
|
"step_time": 43.73763492330909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1178.0,
|
|
"completions/max_terminated_length": 1178.0,
|
|
"completions/mean_length": 804.796875,
|
|
"completions/mean_terminated_length": 804.796875,
|
|
"completions/min_length": 404.0,
|
|
"completions/min_terminated_length": 404.0,
|
|
"entropy": 0.7655727751553059,
|
|
"epoch": 0.5782312925170068,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.006241299764951691,
|
|
"learning_rate": 4.223356009070295e-07,
|
|
"loss": -0.0068,
|
|
"num_tokens": 60357859.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1020,
|
|
"step_time": 41.322896130383015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1335.0,
|
|
"completions/max_terminated_length": 1335.0,
|
|
"completions/mean_length": 757.234375,
|
|
"completions/mean_terminated_length": 757.234375,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.856138713657856,
|
|
"epoch": 0.578798185941043,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.016271464395686053,
|
|
"learning_rate": 4.217687074829932e-07,
|
|
"loss": 0.0125,
|
|
"num_tokens": 60416730.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1021,
|
|
"step_time": 47.48104012850672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1818.0,
|
|
"completions/max_terminated_length": 1818.0,
|
|
"completions/mean_length": 867.625,
|
|
"completions/mean_terminated_length": 867.625,
|
|
"completions/min_length": 418.0,
|
|
"completions/min_terminated_length": 418.0,
|
|
"entropy": 0.6528633162379265,
|
|
"epoch": 0.5793650793650794,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.014403670487808995,
|
|
"learning_rate": 4.212018140589569e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 60483818.0,
|
|
"reward": 0.4453125,
|
|
"reward_std": 0.8023635149002075,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1022,
|
|
"step_time": 64.35244582034647
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1231.0,
|
|
"completions/max_terminated_length": 1231.0,
|
|
"completions/mean_length": 744.484375,
|
|
"completions/mean_terminated_length": 744.484375,
|
|
"completions/min_length": 400.0,
|
|
"completions/min_terminated_length": 400.0,
|
|
"entropy": 0.9489574544131756,
|
|
"epoch": 0.5799319727891157,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.007374967011855915,
|
|
"learning_rate": 4.2063492063492063e-07,
|
|
"loss": 0.047,
|
|
"num_tokens": 60540305.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7811011672019958,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1023,
|
|
"step_time": 43.88332566525787
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 738.359375,
|
|
"completions/mean_terminated_length": 738.359375,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.7529123108834028,
|
|
"epoch": 0.5804988662131519,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6171875,
|
|
"kl": 0.0407306770212017,
|
|
"learning_rate": 4.200680272108844e-07,
|
|
"loss": -0.0204,
|
|
"num_tokens": 60597888.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1024,
|
|
"step_time": 48.967553520575166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1274.0,
|
|
"completions/max_terminated_length": 1274.0,
|
|
"completions/mean_length": 648.015625,
|
|
"completions/mean_terminated_length": 648.015625,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.8017275836318731,
|
|
"epoch": 0.5810657596371882,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6171875,
|
|
"kl": 0.013260460720630363,
|
|
"learning_rate": 4.19501133786848e-07,
|
|
"loss": -0.0045,
|
|
"num_tokens": 60649473.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7489659190177917,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1025,
|
|
"step_time": 45.37248017825186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1942.0,
|
|
"completions/max_terminated_length": 1942.0,
|
|
"completions/mean_length": 793.65625,
|
|
"completions/mean_terminated_length": 793.65625,
|
|
"completions/min_length": 385.0,
|
|
"completions/min_terminated_length": 385.0,
|
|
"entropy": 0.8382871747016907,
|
|
"epoch": 0.5816326530612245,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.005885006976313889,
|
|
"learning_rate": 4.189342403628118e-07,
|
|
"loss": -0.0037,
|
|
"num_tokens": 60711059.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7614454030990601,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1026,
|
|
"step_time": 70.0059653101489
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1418.0,
|
|
"completions/max_terminated_length": 1418.0,
|
|
"completions/mean_length": 732.609375,
|
|
"completions/mean_terminated_length": 732.609375,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.752703320235014,
|
|
"epoch": 0.5821995464852607,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.00898302806308493,
|
|
"learning_rate": 4.183673469387755e-07,
|
|
"loss": 0.0178,
|
|
"num_tokens": 60768226.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1027,
|
|
"step_time": 49.861389183439314
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1628.0,
|
|
"completions/max_terminated_length": 1628.0,
|
|
"completions/mean_length": 782.328125,
|
|
"completions/mean_terminated_length": 782.328125,
|
|
"completions/min_length": 225.0,
|
|
"completions/min_terminated_length": 225.0,
|
|
"entropy": 0.7925093118101358,
|
|
"epoch": 0.5827664399092971,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.008083482200163417,
|
|
"learning_rate": 4.1780045351473917e-07,
|
|
"loss": 0.0251,
|
|
"num_tokens": 60827807.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1028,
|
|
"step_time": 57.28875582199544
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1241.0,
|
|
"completions/max_terminated_length": 1241.0,
|
|
"completions/mean_length": 709.453125,
|
|
"completions/mean_terminated_length": 709.453125,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.8154206275939941,
|
|
"epoch": 0.5833333333333334,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.43359375,
|
|
"kl": 0.016429457173217088,
|
|
"learning_rate": 4.172335600907029e-07,
|
|
"loss": -0.0068,
|
|
"num_tokens": 60888188.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1029,
|
|
"step_time": 46.7139289053157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1555.0,
|
|
"completions/max_terminated_length": 1555.0,
|
|
"completions/mean_length": 796.625,
|
|
"completions/mean_terminated_length": 796.625,
|
|
"completions/min_length": 383.0,
|
|
"completions/min_terminated_length": 383.0,
|
|
"entropy": 0.8111125342547894,
|
|
"epoch": 0.5839002267573696,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.008759330696193501,
|
|
"learning_rate": 4.1666666666666667e-07,
|
|
"loss": -0.0086,
|
|
"num_tokens": 60948852.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7295067310333252,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 1030,
|
|
"step_time": 54.99849047884345
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 714.140625,
|
|
"completions/mean_terminated_length": 714.140625,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.9339536689221859,
|
|
"epoch": 0.5844671201814059,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.359375,
|
|
"kl": 0.008811355975922197,
|
|
"learning_rate": 4.1609977324263036e-07,
|
|
"loss": 0.0014,
|
|
"num_tokens": 61004877.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1031,
|
|
"step_time": 45.26351131219417
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1856.0,
|
|
"completions/max_terminated_length": 1856.0,
|
|
"completions/mean_length": 879.15625,
|
|
"completions/mean_terminated_length": 879.15625,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.5859089009463787,
|
|
"epoch": 0.5850340136054422,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.01350776175968349,
|
|
"learning_rate": 4.1553287981859406e-07,
|
|
"loss": 0.0106,
|
|
"num_tokens": 61071847.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.7216448783874512,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 1032,
|
|
"step_time": 65.91393187083304
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1099.0,
|
|
"completions/max_terminated_length": 1099.0,
|
|
"completions/mean_length": 691.28125,
|
|
"completions/mean_terminated_length": 691.28125,
|
|
"completions/min_length": 286.0,
|
|
"completions/min_terminated_length": 286.0,
|
|
"entropy": 0.9143459908664227,
|
|
"epoch": 0.5856009070294784,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.416015625,
|
|
"kl": 0.010851586994249374,
|
|
"learning_rate": 4.149659863945578e-07,
|
|
"loss": 0.0148,
|
|
"num_tokens": 61126169.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1033,
|
|
"step_time": 38.46899669710547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1340.0,
|
|
"completions/max_terminated_length": 1340.0,
|
|
"completions/mean_length": 725.296875,
|
|
"completions/mean_terminated_length": 725.296875,
|
|
"completions/min_length": 243.0,
|
|
"completions/min_terminated_length": 243.0,
|
|
"entropy": 0.8719802610576153,
|
|
"epoch": 0.5861678004535147,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33984375,
|
|
"kl": 0.03017602207546588,
|
|
"learning_rate": 4.143990929705215e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 61184308.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1034,
|
|
"step_time": 47.88309253565967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1234.0,
|
|
"completions/max_terminated_length": 1234.0,
|
|
"completions/mean_length": 741.96875,
|
|
"completions/mean_terminated_length": 741.96875,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.9664582908153534,
|
|
"epoch": 0.5867346938775511,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.007705768686719239,
|
|
"learning_rate": 4.1383219954648525e-07,
|
|
"loss": 0.0129,
|
|
"num_tokens": 61242882.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1035,
|
|
"step_time": 43.794222825206816
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1165.0,
|
|
"completions/max_terminated_length": 1165.0,
|
|
"completions/mean_length": 717.765625,
|
|
"completions/mean_terminated_length": 717.765625,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.9514351785182953,
|
|
"epoch": 0.5873015873015873,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.00780662419856526,
|
|
"learning_rate": 4.13265306122449e-07,
|
|
"loss": -0.016,
|
|
"num_tokens": 61301027.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1036,
|
|
"step_time": 42.284004542045295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1350.0,
|
|
"completions/max_terminated_length": 1350.0,
|
|
"completions/mean_length": 788.890625,
|
|
"completions/mean_terminated_length": 788.890625,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.8256543353199959,
|
|
"epoch": 0.5878684807256236,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.640625,
|
|
"kl": 0.011924295438802801,
|
|
"learning_rate": 4.1269841269841265e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 61362276.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1037,
|
|
"step_time": 48.0149614168331
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1506.0,
|
|
"completions/max_terminated_length": 1506.0,
|
|
"completions/mean_length": 832.21875,
|
|
"completions/mean_terminated_length": 832.21875,
|
|
"completions/min_length": 365.0,
|
|
"completions/min_terminated_length": 365.0,
|
|
"entropy": 0.6779953818768263,
|
|
"epoch": 0.5884353741496599,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.40625,
|
|
"kl": 0.005162701629160438,
|
|
"learning_rate": 4.121315192743764e-07,
|
|
"loss": 0.0298,
|
|
"num_tokens": 61424434.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1038,
|
|
"step_time": 53.04132007714361
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1947.0,
|
|
"completions/mean_length": 825.078125,
|
|
"completions/mean_terminated_length": 773.1587524414062,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.6513822488486767,
|
|
"epoch": 0.5890022675736961,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.416015625,
|
|
"kl": 0.00839639593323227,
|
|
"learning_rate": 4.1156462585034014e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 61499735.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1039,
|
|
"step_time": 206.40530235879123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1381.0,
|
|
"completions/max_terminated_length": 1381.0,
|
|
"completions/mean_length": 751.15625,
|
|
"completions/mean_terminated_length": 751.15625,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.8985392674803734,
|
|
"epoch": 0.5895691609977324,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.390625,
|
|
"kl": 0.011404858625610359,
|
|
"learning_rate": 4.109977324263038e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 61558145.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1040,
|
|
"step_time": 48.88958250824362
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1174.0,
|
|
"completions/max_terminated_length": 1174.0,
|
|
"completions/mean_length": 699.625,
|
|
"completions/mean_terminated_length": 699.625,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.8420460224151611,
|
|
"epoch": 0.5901360544217688,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.008470737346215174,
|
|
"learning_rate": 4.1043083900226754e-07,
|
|
"loss": -0.0034,
|
|
"num_tokens": 61614545.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1041,
|
|
"step_time": 42.01617934834212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1630.0,
|
|
"completions/max_terminated_length": 1630.0,
|
|
"completions/mean_length": 760.546875,
|
|
"completions/mean_terminated_length": 760.546875,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.5991299413144588,
|
|
"epoch": 0.590702947845805,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.007236314966576174,
|
|
"learning_rate": 4.098639455782313e-07,
|
|
"loss": -0.0038,
|
|
"num_tokens": 61673068.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1042,
|
|
"step_time": 57.953623983077705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 2048.0,
|
|
"completions/mean_length": 819.953125,
|
|
"completions/mean_terminated_length": 819.953125,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.8680130019783974,
|
|
"epoch": 0.5912698412698413,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.0072689572552917525,
|
|
"learning_rate": 4.09297052154195e-07,
|
|
"loss": 0.0249,
|
|
"num_tokens": 61735209.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1043,
|
|
"step_time": 72.51693755481392
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2240.0,
|
|
"completions/max_terminated_length": 2240.0,
|
|
"completions/mean_length": 903.1875,
|
|
"completions/mean_terminated_length": 903.1875,
|
|
"completions/min_length": 478.0,
|
|
"completions/min_terminated_length": 478.0,
|
|
"entropy": 0.740302350372076,
|
|
"epoch": 0.5918367346938775,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.00869008646986913,
|
|
"learning_rate": 4.087301587301587e-07,
|
|
"loss": 0.0372,
|
|
"num_tokens": 61802949.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1044,
|
|
"step_time": 80.77024371083826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1147.0,
|
|
"completions/max_terminated_length": 1147.0,
|
|
"completions/mean_length": 613.140625,
|
|
"completions/mean_terminated_length": 613.140625,
|
|
"completions/min_length": 172.0,
|
|
"completions/min_terminated_length": 172.0,
|
|
"entropy": 0.9890522547066212,
|
|
"epoch": 0.5924036281179138,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.66796875,
|
|
"kl": 0.014456457502092235,
|
|
"learning_rate": 4.0816326530612243e-07,
|
|
"loss": 0.0298,
|
|
"num_tokens": 61855438.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1045,
|
|
"step_time": 41.359472713433206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1409.0,
|
|
"completions/max_terminated_length": 1409.0,
|
|
"completions/mean_length": 757.640625,
|
|
"completions/mean_terminated_length": 757.640625,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.4957543648779392,
|
|
"epoch": 0.5929705215419501,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.006583279755432159,
|
|
"learning_rate": 4.075963718820862e-07,
|
|
"loss": 0.0206,
|
|
"num_tokens": 61914247.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1046,
|
|
"step_time": 49.771801337599754
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1201.0,
|
|
"completions/max_terminated_length": 1201.0,
|
|
"completions/mean_length": 622.6875,
|
|
"completions/mean_terminated_length": 622.6875,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.9657610915601254,
|
|
"epoch": 0.5935374149659864,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6484375,
|
|
"kl": 0.008885723756975494,
|
|
"learning_rate": 4.0702947845804987e-07,
|
|
"loss": 0.0199,
|
|
"num_tokens": 61962403.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.854493260383606,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1047,
|
|
"step_time": 42.102924117818475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1824.0,
|
|
"completions/max_terminated_length": 1824.0,
|
|
"completions/mean_length": 830.390625,
|
|
"completions/mean_terminated_length": 830.390625,
|
|
"completions/min_length": 307.0,
|
|
"completions/min_terminated_length": 307.0,
|
|
"entropy": 0.5310399085283279,
|
|
"epoch": 0.5941043083900227,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.009403905656654388,
|
|
"learning_rate": 4.0646258503401357e-07,
|
|
"loss": 0.0507,
|
|
"num_tokens": 62025732.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1048,
|
|
"step_time": 65.35518271382898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1495.0,
|
|
"completions/max_terminated_length": 1495.0,
|
|
"completions/mean_length": 727.796875,
|
|
"completions/mean_terminated_length": 727.796875,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 0.9522303361445665,
|
|
"epoch": 0.594671201814059,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.3203125,
|
|
"kl": 0.007637298083864152,
|
|
"learning_rate": 4.058956916099773e-07,
|
|
"loss": 0.0063,
|
|
"num_tokens": 62083199.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1049,
|
|
"step_time": 52.62421330064535
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1436.0,
|
|
"completions/max_terminated_length": 1436.0,
|
|
"completions/mean_length": 786.53125,
|
|
"completions/mean_terminated_length": 786.53125,
|
|
"completions/min_length": 274.0,
|
|
"completions/min_terminated_length": 274.0,
|
|
"entropy": 0.6645113658159971,
|
|
"epoch": 0.5952380952380952,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.006322235392872244,
|
|
"learning_rate": 4.05328798185941e-07,
|
|
"loss": 0.0331,
|
|
"num_tokens": 62143097.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1050,
|
|
"step_time": 50.04692924488336
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1468.0,
|
|
"completions/max_terminated_length": 1468.0,
|
|
"completions/mean_length": 795.671875,
|
|
"completions/mean_terminated_length": 795.671875,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.7811813168227673,
|
|
"epoch": 0.5958049886621315,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4453125,
|
|
"kl": 0.01016502054699231,
|
|
"learning_rate": 4.0476190476190476e-07,
|
|
"loss": -0.0043,
|
|
"num_tokens": 62204604.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1051,
|
|
"step_time": 52.65343519579619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1696.0,
|
|
"completions/max_terminated_length": 1696.0,
|
|
"completions/mean_length": 767.3125,
|
|
"completions/mean_terminated_length": 767.3125,
|
|
"completions/min_length": 220.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"entropy": 0.8036662619560957,
|
|
"epoch": 0.5963718820861678,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.008400273902225308,
|
|
"learning_rate": 4.0419501133786846e-07,
|
|
"loss": -0.0167,
|
|
"num_tokens": 62264864.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1052,
|
|
"step_time": 60.46609304379672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1431.0,
|
|
"completions/max_terminated_length": 1431.0,
|
|
"completions/mean_length": 763.921875,
|
|
"completions/mean_terminated_length": 763.921875,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.6650191079825163,
|
|
"epoch": 0.5969387755102041,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.01008718907542061,
|
|
"learning_rate": 4.0362811791383216e-07,
|
|
"loss": 0.015,
|
|
"num_tokens": 62323523.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1053,
|
|
"step_time": 50.73767376691103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1643.0,
|
|
"completions/max_terminated_length": 1643.0,
|
|
"completions/mean_length": 765.78125,
|
|
"completions/mean_terminated_length": 765.78125,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.8426890708506107,
|
|
"epoch": 0.5975056689342404,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.68359375,
|
|
"kl": 0.009880403522402048,
|
|
"learning_rate": 4.030612244897959e-07,
|
|
"loss": 0.0809,
|
|
"num_tokens": 62382221.0,
|
|
"reward": 0.484375,
|
|
"reward_std": 0.8211913108825684,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1054,
|
|
"step_time": 57.73382254317403
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1528.0,
|
|
"completions/max_terminated_length": 1528.0,
|
|
"completions/mean_length": 778.65625,
|
|
"completions/mean_terminated_length": 778.65625,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.5825038570910692,
|
|
"epoch": 0.5980725623582767,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.013963502191472799,
|
|
"learning_rate": 4.0249433106575965e-07,
|
|
"loss": 0.0296,
|
|
"num_tokens": 62443639.0,
|
|
"reward": 0.53125,
|
|
"reward_std": 0.9079673886299133,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1055,
|
|
"step_time": 54.702407508157194
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1293.0,
|
|
"completions/max_terminated_length": 1293.0,
|
|
"completions/mean_length": 756.390625,
|
|
"completions/mean_terminated_length": 756.390625,
|
|
"completions/min_length": 45.0,
|
|
"completions/min_terminated_length": 45.0,
|
|
"entropy": 0.7905134595930576,
|
|
"epoch": 0.5986394557823129,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.60546875,
|
|
"kl": 0.008505104589858092,
|
|
"learning_rate": 4.019274376417233e-07,
|
|
"loss": 0.019,
|
|
"num_tokens": 62502576.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1056,
|
|
"step_time": 45.828205581754446
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1324.0,
|
|
"completions/max_terminated_length": 1324.0,
|
|
"completions/mean_length": 779.90625,
|
|
"completions/mean_terminated_length": 779.90625,
|
|
"completions/min_length": 256.0,
|
|
"completions/min_terminated_length": 256.0,
|
|
"entropy": 0.6706081181764603,
|
|
"epoch": 0.5992063492063492,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.455078125,
|
|
"kl": 0.015872819727519527,
|
|
"learning_rate": 4.0136054421768705e-07,
|
|
"loss": -0.0078,
|
|
"num_tokens": 62561994.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.8011648058891296,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1057,
|
|
"step_time": 46.46486928779632
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1737.0,
|
|
"completions/max_terminated_length": 1737.0,
|
|
"completions/mean_length": 770.359375,
|
|
"completions/mean_terminated_length": 770.359375,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 0.7864743266254663,
|
|
"epoch": 0.5997732426303855,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.008506573140039109,
|
|
"learning_rate": 4.007936507936508e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 62621281.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1058,
|
|
"step_time": 61.91380777955055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1497.0,
|
|
"completions/max_terminated_length": 1497.0,
|
|
"completions/mean_length": 775.9375,
|
|
"completions/mean_terminated_length": 775.9375,
|
|
"completions/min_length": 175.0,
|
|
"completions/min_terminated_length": 175.0,
|
|
"entropy": 0.7870574370026588,
|
|
"epoch": 0.6003401360544217,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.0071375035477103665,
|
|
"learning_rate": 4.002267573696145e-07,
|
|
"loss": -0.0012,
|
|
"num_tokens": 62681693.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1059,
|
|
"step_time": 52.61389343440533
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1635.0,
|
|
"completions/max_terminated_length": 1635.0,
|
|
"completions/mean_length": 805.421875,
|
|
"completions/mean_terminated_length": 805.421875,
|
|
"completions/min_length": 325.0,
|
|
"completions/min_terminated_length": 325.0,
|
|
"entropy": 0.813016552478075,
|
|
"epoch": 0.6009070294784581,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.0060426656127674505,
|
|
"learning_rate": 3.996598639455782e-07,
|
|
"loss": -0.0358,
|
|
"num_tokens": 62744448.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.8261595964431763,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1060,
|
|
"step_time": 57.79994550254196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1296.0,
|
|
"completions/max_terminated_length": 1296.0,
|
|
"completions/mean_length": 832.84375,
|
|
"completions/mean_terminated_length": 832.84375,
|
|
"completions/min_length": 417.0,
|
|
"completions/min_terminated_length": 417.0,
|
|
"entropy": 0.6297138258814812,
|
|
"epoch": 0.6014739229024944,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.009992541221436113,
|
|
"learning_rate": 3.9909297052154194e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 62808798.0,
|
|
"reward": 0.4765625,
|
|
"reward_std": 0.8234156966209412,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1061,
|
|
"step_time": 45.78222487960011
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1325.0,
|
|
"completions/max_terminated_length": 1325.0,
|
|
"completions/mean_length": 677.984375,
|
|
"completions/mean_terminated_length": 677.984375,
|
|
"completions/min_length": 413.0,
|
|
"completions/min_terminated_length": 413.0,
|
|
"entropy": 0.7807711362838745,
|
|
"epoch": 0.6020408163265306,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.02023360281600617,
|
|
"learning_rate": 3.9852607709750563e-07,
|
|
"loss": -0.0061,
|
|
"num_tokens": 62861565.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.7466442584991455,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1062,
|
|
"step_time": 46.807525185868144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1193.0,
|
|
"completions/max_terminated_length": 1193.0,
|
|
"completions/mean_length": 709.984375,
|
|
"completions/mean_terminated_length": 709.984375,
|
|
"completions/min_length": 430.0,
|
|
"completions/min_terminated_length": 430.0,
|
|
"entropy": 0.6286953091621399,
|
|
"epoch": 0.6026077097505669,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.015123460529139265,
|
|
"learning_rate": 3.979591836734694e-07,
|
|
"loss": 0.0109,
|
|
"num_tokens": 62917980.0,
|
|
"reward": 0.5234375,
|
|
"reward_std": 0.8927827477455139,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1063,
|
|
"step_time": 42.48129607271403
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1246.0,
|
|
"completions/max_terminated_length": 1246.0,
|
|
"completions/mean_length": 708.828125,
|
|
"completions/mean_terminated_length": 708.828125,
|
|
"completions/min_length": 240.0,
|
|
"completions/min_terminated_length": 240.0,
|
|
"entropy": 0.7838305328041315,
|
|
"epoch": 0.6031746031746031,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.009680385657702573,
|
|
"learning_rate": 3.973922902494331e-07,
|
|
"loss": -0.0103,
|
|
"num_tokens": 62973473.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1064,
|
|
"step_time": 44.255540791898966
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1482.0,
|
|
"completions/max_terminated_length": 1482.0,
|
|
"completions/mean_length": 797.71875,
|
|
"completions/mean_terminated_length": 797.71875,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 0.8374659940600395,
|
|
"epoch": 0.6037414965986394,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"kl": 0.00810998395900242,
|
|
"learning_rate": 3.968253968253968e-07,
|
|
"loss": -0.0054,
|
|
"num_tokens": 63036215.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.7466442584991455,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1065,
|
|
"step_time": 53.234934319742024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1537.0,
|
|
"completions/max_terminated_length": 1537.0,
|
|
"completions/mean_length": 928.109375,
|
|
"completions/mean_terminated_length": 928.109375,
|
|
"completions/min_length": 406.0,
|
|
"completions/min_terminated_length": 406.0,
|
|
"entropy": 0.6840494628995657,
|
|
"epoch": 0.6043083900226758,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.009237834572559223,
|
|
"learning_rate": 3.962585034013605e-07,
|
|
"loss": -0.0152,
|
|
"num_tokens": 63106470.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1066,
|
|
"step_time": 54.04610664304346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1488.0,
|
|
"completions/max_terminated_length": 1488.0,
|
|
"completions/mean_length": 768.5,
|
|
"completions/mean_terminated_length": 768.5,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 0.7869984470307827,
|
|
"epoch": 0.604875283446712,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.60546875,
|
|
"kl": 0.011371919506927952,
|
|
"learning_rate": 3.956916099773243e-07,
|
|
"loss": 0.0186,
|
|
"num_tokens": 63169014.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1067,
|
|
"step_time": 54.05696022696793
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1055.0,
|
|
"completions/max_terminated_length": 1055.0,
|
|
"completions/mean_length": 698.8125,
|
|
"completions/mean_terminated_length": 698.8125,
|
|
"completions/min_length": 385.0,
|
|
"completions/min_terminated_length": 385.0,
|
|
"entropy": 0.8471840135753155,
|
|
"epoch": 0.6054421768707483,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.015613438474247232,
|
|
"learning_rate": 3.951247165532879e-07,
|
|
"loss": 0.0165,
|
|
"num_tokens": 63224730.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6029338836669922,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1068,
|
|
"step_time": 37.711398063227534
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1840.0,
|
|
"completions/max_terminated_length": 1840.0,
|
|
"completions/mean_length": 733.65625,
|
|
"completions/mean_terminated_length": 733.65625,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 0.6620960850268602,
|
|
"epoch": 0.6060090702947846,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6953125,
|
|
"kl": 0.01109703682595864,
|
|
"learning_rate": 3.9455782312925167e-07,
|
|
"loss": 0.0104,
|
|
"num_tokens": 63281708.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8188852071762085,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1069,
|
|
"step_time": 65.03774888534099
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1619.0,
|
|
"completions/max_terminated_length": 1619.0,
|
|
"completions/mean_length": 645.375,
|
|
"completions/mean_terminated_length": 645.375,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 1.0387386474758387,
|
|
"epoch": 0.6065759637188208,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.007418462104396895,
|
|
"learning_rate": 3.939909297052154e-07,
|
|
"loss": -0.0482,
|
|
"num_tokens": 63333508.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1070,
|
|
"step_time": 56.99738501012325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1450.0,
|
|
"completions/max_terminated_length": 1450.0,
|
|
"completions/mean_length": 866.109375,
|
|
"completions/mean_terminated_length": 866.109375,
|
|
"completions/min_length": 448.0,
|
|
"completions/min_terminated_length": 448.0,
|
|
"entropy": 0.73966334015131,
|
|
"epoch": 0.6071428571428571,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.008390052273171023,
|
|
"learning_rate": 3.934240362811791e-07,
|
|
"loss": 0.0406,
|
|
"num_tokens": 63399171.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1071,
|
|
"step_time": 51.70856096316129
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1828.0,
|
|
"completions/max_terminated_length": 1828.0,
|
|
"completions/mean_length": 849.84375,
|
|
"completions/mean_terminated_length": 849.84375,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 0.8376497514545918,
|
|
"epoch": 0.6077097505668935,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.008251386199845001,
|
|
"learning_rate": 3.928571428571428e-07,
|
|
"loss": 0.0064,
|
|
"num_tokens": 63465449.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1072,
|
|
"step_time": 64.93658199440688
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1130.0,
|
|
"completions/max_terminated_length": 1130.0,
|
|
"completions/mean_length": 711.109375,
|
|
"completions/mean_terminated_length": 711.109375,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.7102722302079201,
|
|
"epoch": 0.6082766439909297,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.015092990099219605,
|
|
"learning_rate": 3.9229024943310656e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 63522200.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1073,
|
|
"step_time": 40.60517934337258
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1472.0,
|
|
"completions/max_terminated_length": 1472.0,
|
|
"completions/mean_length": 755.234375,
|
|
"completions/mean_terminated_length": 755.234375,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.8104569092392921,
|
|
"epoch": 0.608843537414966,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5390625,
|
|
"kl": 0.011562505053007044,
|
|
"learning_rate": 3.9172335600907025e-07,
|
|
"loss": 0.0263,
|
|
"num_tokens": 63579495.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1074,
|
|
"step_time": 51.92296173609793
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1547.0,
|
|
"completions/max_terminated_length": 1547.0,
|
|
"completions/mean_length": 672.265625,
|
|
"completions/mean_terminated_length": 672.265625,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.6472252905368805,
|
|
"epoch": 0.6094104308390023,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.007816172714228742,
|
|
"learning_rate": 3.91156462585034e-07,
|
|
"loss": -0.0136,
|
|
"num_tokens": 63632960.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1075,
|
|
"step_time": 54.468455894850194
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1365.0,
|
|
"completions/max_terminated_length": 1365.0,
|
|
"completions/mean_length": 878.109375,
|
|
"completions/mean_terminated_length": 878.109375,
|
|
"completions/min_length": 351.0,
|
|
"completions/min_terminated_length": 351.0,
|
|
"entropy": 0.5308348014950752,
|
|
"epoch": 0.6099773242630385,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.011981691131950356,
|
|
"learning_rate": 3.905895691609977e-07,
|
|
"loss": 0.0339,
|
|
"num_tokens": 63699687.0,
|
|
"reward": 0.6171875,
|
|
"reward_std": 0.8852503299713135,
|
|
"rewards/accuracy_reward_func/mean": 0.5625,
|
|
"rewards/accuracy_reward_func/std": 0.9063270092010498,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1076,
|
|
"step_time": 48.096376333385706
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1272.0,
|
|
"completions/max_terminated_length": 1272.0,
|
|
"completions/mean_length": 750.546875,
|
|
"completions/mean_terminated_length": 750.546875,
|
|
"completions/min_length": 378.0,
|
|
"completions/min_terminated_length": 378.0,
|
|
"entropy": 0.820930752903223,
|
|
"epoch": 0.6105442176870748,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.007634779685758986,
|
|
"learning_rate": 3.9002267573696145e-07,
|
|
"loss": 0.0004,
|
|
"num_tokens": 63758306.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6154024600982666,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 1077,
|
|
"step_time": 45.158872643485665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3823.0,
|
|
"completions/mean_length": 961.75,
|
|
"completions/mean_terminated_length": 860.6451416015625,
|
|
"completions/min_length": 391.0,
|
|
"completions/min_terminated_length": 391.0,
|
|
"entropy": 0.6471217703074217,
|
|
"epoch": 0.6111111111111112,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.0054104540758999065,
|
|
"learning_rate": 3.8945578231292515e-07,
|
|
"loss": -0.0034,
|
|
"num_tokens": 63830170.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1078,
|
|
"step_time": 171.27073407080024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1425.0,
|
|
"completions/max_terminated_length": 1425.0,
|
|
"completions/mean_length": 723.59375,
|
|
"completions/mean_terminated_length": 723.59375,
|
|
"completions/min_length": 203.0,
|
|
"completions/min_terminated_length": 203.0,
|
|
"entropy": 0.8422863073647022,
|
|
"epoch": 0.6116780045351474,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6875,
|
|
"kl": 0.008222233387641609,
|
|
"learning_rate": 3.888888888888889e-07,
|
|
"loss": -0.0461,
|
|
"num_tokens": 63888208.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7708735466003418,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1079,
|
|
"step_time": 50.33240020275116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1123.0,
|
|
"completions/max_terminated_length": 1123.0,
|
|
"completions/mean_length": 639.328125,
|
|
"completions/mean_terminated_length": 639.328125,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"entropy": 0.8335273284465075,
|
|
"epoch": 0.6122448979591837,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.012569834885653108,
|
|
"learning_rate": 3.883219954648526e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 63938357.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1080,
|
|
"step_time": 39.52465805597603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2297.0,
|
|
"completions/max_terminated_length": 2297.0,
|
|
"completions/mean_length": 742.046875,
|
|
"completions/mean_terminated_length": 742.046875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 1.004751643165946,
|
|
"epoch": 0.61281179138322,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"kl": 0.009632715853513218,
|
|
"learning_rate": 3.877551020408163e-07,
|
|
"loss": -0.0479,
|
|
"num_tokens": 63995016.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1081,
|
|
"step_time": 83.10484649520367
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 753.40625,
|
|
"completions/mean_terminated_length": 753.40625,
|
|
"completions/min_length": 268.0,
|
|
"completions/min_terminated_length": 268.0,
|
|
"entropy": 0.8771509099751711,
|
|
"epoch": 0.6133786848072562,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.011041131539968774,
|
|
"learning_rate": 3.8718820861678004e-07,
|
|
"loss": -0.0112,
|
|
"num_tokens": 64053434.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1082,
|
|
"step_time": 48.77529868669808
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1390.0,
|
|
"completions/max_terminated_length": 1390.0,
|
|
"completions/mean_length": 685.765625,
|
|
"completions/mean_terminated_length": 685.765625,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 0.8738362230360508,
|
|
"epoch": 0.6139455782312925,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.010783211982925422,
|
|
"learning_rate": 3.866213151927438e-07,
|
|
"loss": 0.0458,
|
|
"num_tokens": 64109883.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1083,
|
|
"step_time": 49.97598290350288
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1735.0,
|
|
"completions/max_terminated_length": 1735.0,
|
|
"completions/mean_length": 794.40625,
|
|
"completions/mean_terminated_length": 794.40625,
|
|
"completions/min_length": 379.0,
|
|
"completions/min_terminated_length": 379.0,
|
|
"entropy": 0.8340683802962303,
|
|
"epoch": 0.6145124716553289,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.009540995059069246,
|
|
"learning_rate": 3.8605442176870743e-07,
|
|
"loss": -0.0377,
|
|
"num_tokens": 64174069.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1084,
|
|
"step_time": 61.9020598847419
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1133.0,
|
|
"completions/max_terminated_length": 1133.0,
|
|
"completions/mean_length": 744.859375,
|
|
"completions/mean_terminated_length": 744.859375,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.7178486529737711,
|
|
"epoch": 0.6150793650793651,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.00786908544250764,
|
|
"learning_rate": 3.854875283446712e-07,
|
|
"loss": 0.0246,
|
|
"num_tokens": 64234972.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1085,
|
|
"step_time": 41.07743614260107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1245.0,
|
|
"completions/max_terminated_length": 1245.0,
|
|
"completions/mean_length": 772.609375,
|
|
"completions/mean_terminated_length": 772.609375,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.7387854773551226,
|
|
"epoch": 0.6156462585034014,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.008304751638206653,
|
|
"learning_rate": 3.8492063492063493e-07,
|
|
"loss": 0.0014,
|
|
"num_tokens": 64294715.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1086,
|
|
"step_time": 43.84017410874367
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1181.0,
|
|
"completions/max_terminated_length": 1181.0,
|
|
"completions/mean_length": 656.40625,
|
|
"completions/mean_terminated_length": 656.40625,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.8134062401950359,
|
|
"epoch": 0.6162131519274376,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.009632436544052325,
|
|
"learning_rate": 3.843537414965986e-07,
|
|
"loss": 0.0116,
|
|
"num_tokens": 64345709.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7194505929946899,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1087,
|
|
"step_time": 41.0964750116691
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1600.0,
|
|
"completions/max_terminated_length": 1600.0,
|
|
"completions/mean_length": 742.015625,
|
|
"completions/mean_terminated_length": 742.015625,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.9605098329484463,
|
|
"epoch": 0.6167800453514739,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.011273718293523416,
|
|
"learning_rate": 3.837868480725623e-07,
|
|
"loss": -0.011,
|
|
"num_tokens": 64404110.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1088,
|
|
"step_time": 55.793396341614425
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1479.0,
|
|
"completions/max_terminated_length": 1479.0,
|
|
"completions/mean_length": 660.390625,
|
|
"completions/mean_terminated_length": 660.390625,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.8606309406459332,
|
|
"epoch": 0.6173469387755102,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.016010076826205477,
|
|
"learning_rate": 3.8321995464852607e-07,
|
|
"loss": -0.0106,
|
|
"num_tokens": 64456823.0,
|
|
"reward": 0.46875,
|
|
"reward_std": 0.8255589604377747,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1089,
|
|
"step_time": 53.01323266979307
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1400.0,
|
|
"completions/max_terminated_length": 1400.0,
|
|
"completions/mean_length": 708.09375,
|
|
"completions/mean_terminated_length": 708.09375,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 0.837235389277339,
|
|
"epoch": 0.6179138321995464,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.011954975896514952,
|
|
"learning_rate": 3.8265306122448977e-07,
|
|
"loss": -0.0007,
|
|
"num_tokens": 64513717.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1090,
|
|
"step_time": 49.511431032791734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1481.0,
|
|
"completions/max_terminated_length": 1481.0,
|
|
"completions/mean_length": 734.015625,
|
|
"completions/mean_terminated_length": 734.015625,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.9118502270430326,
|
|
"epoch": 0.6184807256235828,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.007842142826120835,
|
|
"learning_rate": 3.820861678004535e-07,
|
|
"loss": 0.0097,
|
|
"num_tokens": 64570166.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1091,
|
|
"step_time": 51.94239405822009
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1364.0,
|
|
"completions/max_terminated_length": 1364.0,
|
|
"completions/mean_length": 734.3125,
|
|
"completions/mean_terminated_length": 734.3125,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"entropy": 0.7291359808295965,
|
|
"epoch": 0.6190476190476191,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.00735489338694606,
|
|
"learning_rate": 3.815192743764172e-07,
|
|
"loss": 0.018,
|
|
"num_tokens": 64627634.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7867957949638367,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1092,
|
|
"step_time": 48.54821335244924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1260.0,
|
|
"completions/max_terminated_length": 1260.0,
|
|
"completions/mean_length": 672.5625,
|
|
"completions/mean_terminated_length": 672.5625,
|
|
"completions/min_length": 171.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"entropy": 0.9438561089336872,
|
|
"epoch": 0.6196145124716553,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2470703125,
|
|
"kl": 0.006280550485826097,
|
|
"learning_rate": 3.809523809523809e-07,
|
|
"loss": -0.0036,
|
|
"num_tokens": 64679430.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1093,
|
|
"step_time": 44.567545872181654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1258.0,
|
|
"completions/max_terminated_length": 1258.0,
|
|
"completions/mean_length": 681.125,
|
|
"completions/mean_terminated_length": 681.125,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.9465801119804382,
|
|
"epoch": 0.6201814058956916,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.80859375,
|
|
"kl": 0.006022705638315529,
|
|
"learning_rate": 3.8038548752834466e-07,
|
|
"loss": -0.0071,
|
|
"num_tokens": 64732158.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1094,
|
|
"step_time": 44.596594678238034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2835.0,
|
|
"completions/max_terminated_length": 2835.0,
|
|
"completions/mean_length": 732.25,
|
|
"completions/mean_terminated_length": 732.25,
|
|
"completions/min_length": 307.0,
|
|
"completions/min_terminated_length": 307.0,
|
|
"entropy": 0.8135250508785248,
|
|
"epoch": 0.6207482993197279,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.015079224889632314,
|
|
"learning_rate": 3.798185941043084e-07,
|
|
"loss": 0.0304,
|
|
"num_tokens": 64788358.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1095,
|
|
"step_time": 105.4008199069649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1311.0,
|
|
"completions/max_terminated_length": 1311.0,
|
|
"completions/mean_length": 694.96875,
|
|
"completions/mean_terminated_length": 694.96875,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 1.0938508100807667,
|
|
"epoch": 0.6213151927437641,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.31640625,
|
|
"kl": 0.007630687454366125,
|
|
"learning_rate": 3.7925170068027205e-07,
|
|
"loss": 0.0004,
|
|
"num_tokens": 64842444.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1096,
|
|
"step_time": 45.90716629475355
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1442.0,
|
|
"completions/max_terminated_length": 1442.0,
|
|
"completions/mean_length": 858.46875,
|
|
"completions/mean_terminated_length": 858.46875,
|
|
"completions/min_length": 438.0,
|
|
"completions/min_terminated_length": 438.0,
|
|
"entropy": 0.7014495991170406,
|
|
"epoch": 0.6218820861678005,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.39453125,
|
|
"kl": 0.00925961225584615,
|
|
"learning_rate": 3.786848072562358e-07,
|
|
"loss": 0.0069,
|
|
"num_tokens": 64907842.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1097,
|
|
"step_time": 50.802561552263796
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1500.0,
|
|
"completions/max_terminated_length": 1500.0,
|
|
"completions/mean_length": 696.359375,
|
|
"completions/mean_terminated_length": 696.359375,
|
|
"completions/min_length": 238.0,
|
|
"completions/min_terminated_length": 238.0,
|
|
"entropy": 0.8630800936371088,
|
|
"epoch": 0.6224489795918368,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2451171875,
|
|
"kl": 0.04774674313375726,
|
|
"learning_rate": 3.7811791383219955e-07,
|
|
"loss": -0.0112,
|
|
"num_tokens": 64962417.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1098,
|
|
"step_time": 52.753368044272065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1415.0,
|
|
"completions/max_terminated_length": 1415.0,
|
|
"completions/mean_length": 723.484375,
|
|
"completions/mean_terminated_length": 723.484375,
|
|
"completions/min_length": 351.0,
|
|
"completions/min_terminated_length": 351.0,
|
|
"entropy": 0.9499944560229778,
|
|
"epoch": 0.623015873015873,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.01397705078125,
|
|
"kl": 0.010864733339985833,
|
|
"learning_rate": 3.7755102040816324e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 65019696.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.1666666716337204,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1099,
|
|
"step_time": 50.37411595694721
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1354.0,
|
|
"completions/max_terminated_length": 1354.0,
|
|
"completions/mean_length": 719.171875,
|
|
"completions/mean_terminated_length": 719.171875,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 0.8007517084479332,
|
|
"epoch": 0.6235827664399093,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.008993162467959337,
|
|
"learning_rate": 3.7698412698412694e-07,
|
|
"loss": 0.0376,
|
|
"num_tokens": 65075195.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.8140628933906555,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1100,
|
|
"step_time": 47.34152162447572
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1719.0,
|
|
"completions/max_terminated_length": 1719.0,
|
|
"completions/mean_length": 591.078125,
|
|
"completions/mean_terminated_length": 591.078125,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.7563750799745321,
|
|
"epoch": 0.6241496598639455,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.01614830840844661,
|
|
"learning_rate": 3.764172335600907e-07,
|
|
"loss": 0.0027,
|
|
"num_tokens": 65127544.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1101,
|
|
"step_time": 62.61190635804087
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1190.0,
|
|
"completions/max_terminated_length": 1190.0,
|
|
"completions/mean_length": 724.65625,
|
|
"completions/mean_terminated_length": 724.65625,
|
|
"completions/min_length": 201.0,
|
|
"completions/min_terminated_length": 201.0,
|
|
"entropy": 0.8077904116362333,
|
|
"epoch": 0.6247165532879818,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.42578125,
|
|
"kl": 0.008323072484927252,
|
|
"learning_rate": 3.758503401360544e-07,
|
|
"loss": -0.0124,
|
|
"num_tokens": 65185210.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1102,
|
|
"step_time": 42.44041490089148
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1205.0,
|
|
"completions/max_terminated_length": 1205.0,
|
|
"completions/mean_length": 732.828125,
|
|
"completions/mean_terminated_length": 732.828125,
|
|
"completions/min_length": 193.0,
|
|
"completions/min_terminated_length": 193.0,
|
|
"entropy": 0.8504715263843536,
|
|
"epoch": 0.6252834467120182,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.009405125980265439,
|
|
"learning_rate": 3.7528344671201813e-07,
|
|
"loss": 0.0191,
|
|
"num_tokens": 65242055.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1103,
|
|
"step_time": 43.0524281328544
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1289.0,
|
|
"completions/max_terminated_length": 1289.0,
|
|
"completions/mean_length": 758.71875,
|
|
"completions/mean_terminated_length": 758.71875,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.6762850880622864,
|
|
"epoch": 0.6258503401360545,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.009180467139231041,
|
|
"learning_rate": 3.7471655328798183e-07,
|
|
"loss": -0.0136,
|
|
"num_tokens": 65300869.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4787135720252991,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1104,
|
|
"step_time": 46.34762056544423
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1637.0,
|
|
"completions/max_terminated_length": 1637.0,
|
|
"completions/mean_length": 772.8125,
|
|
"completions/mean_terminated_length": 772.8125,
|
|
"completions/min_length": 287.0,
|
|
"completions/min_terminated_length": 287.0,
|
|
"entropy": 0.8533004373311996,
|
|
"epoch": 0.6264172335600907,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.011492704870761372,
|
|
"learning_rate": 3.741496598639456e-07,
|
|
"loss": 0.0108,
|
|
"num_tokens": 65363361.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1105,
|
|
"step_time": 59.60923634842038
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1953.0,
|
|
"completions/max_terminated_length": 1953.0,
|
|
"completions/mean_length": 814.671875,
|
|
"completions/mean_terminated_length": 814.671875,
|
|
"completions/min_length": 325.0,
|
|
"completions/min_terminated_length": 325.0,
|
|
"entropy": 0.71065310575068,
|
|
"epoch": 0.626984126984127,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.009234372701030225,
|
|
"learning_rate": 3.735827664399093e-07,
|
|
"loss": 0.0335,
|
|
"num_tokens": 65425604.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1106,
|
|
"step_time": 69.39124313555658
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1662.0,
|
|
"completions/max_terminated_length": 1662.0,
|
|
"completions/mean_length": 718.890625,
|
|
"completions/mean_terminated_length": 718.890625,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.8713082894682884,
|
|
"epoch": 0.6275510204081632,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.69140625,
|
|
"kl": 0.010752781498013064,
|
|
"learning_rate": 3.73015873015873e-07,
|
|
"loss": -0.0369,
|
|
"num_tokens": 65481237.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1107,
|
|
"step_time": 58.43307991698384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 713.5,
|
|
"completions/mean_terminated_length": 713.5,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.7141605261713266,
|
|
"epoch": 0.6281179138321995,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.009599166587577201,
|
|
"learning_rate": 3.724489795918367e-07,
|
|
"loss": 0.0213,
|
|
"num_tokens": 65537477.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1108,
|
|
"step_time": 48.0826120832935
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1697.0,
|
|
"completions/max_terminated_length": 1697.0,
|
|
"completions/mean_length": 847.9375,
|
|
"completions/mean_terminated_length": 847.9375,
|
|
"completions/min_length": 380.0,
|
|
"completions/min_terminated_length": 380.0,
|
|
"entropy": 0.8109442666172981,
|
|
"epoch": 0.6286848072562359,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.00913382729049772,
|
|
"learning_rate": 3.718820861678004e-07,
|
|
"loss": -0.0038,
|
|
"num_tokens": 65601521.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1109,
|
|
"step_time": 59.80372723378241
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1492.0,
|
|
"completions/max_terminated_length": 1492.0,
|
|
"completions/mean_length": 797.203125,
|
|
"completions/mean_terminated_length": 797.203125,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.7242512535303831,
|
|
"epoch": 0.6292517006802721,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.291015625,
|
|
"kl": 0.0118995321681723,
|
|
"learning_rate": 3.7131519274376417e-07,
|
|
"loss": 0.0118,
|
|
"num_tokens": 65662110.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1110,
|
|
"step_time": 52.27861657273024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1199.0,
|
|
"completions/max_terminated_length": 1199.0,
|
|
"completions/mean_length": 728.75,
|
|
"completions/mean_terminated_length": 728.75,
|
|
"completions/min_length": 360.0,
|
|
"completions/min_terminated_length": 360.0,
|
|
"entropy": 0.9125937074422836,
|
|
"epoch": 0.6298185941043084,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.008830365215544589,
|
|
"learning_rate": 3.707482993197279e-07,
|
|
"loss": 0.0106,
|
|
"num_tokens": 65720526.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 1111,
|
|
"step_time": 42.55936660896987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1329.0,
|
|
"completions/max_terminated_length": 1329.0,
|
|
"completions/mean_length": 723.34375,
|
|
"completions/mean_terminated_length": 723.34375,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.7338997051119804,
|
|
"epoch": 0.6303854875283447,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.007963202777318656,
|
|
"learning_rate": 3.7018140589569156e-07,
|
|
"loss": 0.0049,
|
|
"num_tokens": 65776924.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.835710883140564,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1112,
|
|
"step_time": 47.28655724320561
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1428.0,
|
|
"completions/max_terminated_length": 1428.0,
|
|
"completions/mean_length": 640.828125,
|
|
"completions/mean_terminated_length": 640.828125,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 0.8421805556863546,
|
|
"epoch": 0.6309523809523809,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.01376878161681816,
|
|
"learning_rate": 3.696145124716553e-07,
|
|
"loss": -0.0053,
|
|
"num_tokens": 65830929.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1113,
|
|
"step_time": 50.82762067951262
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1500.0,
|
|
"completions/max_terminated_length": 1500.0,
|
|
"completions/mean_length": 754.703125,
|
|
"completions/mean_terminated_length": 754.703125,
|
|
"completions/min_length": 410.0,
|
|
"completions/min_terminated_length": 410.0,
|
|
"entropy": 0.8844935037195683,
|
|
"epoch": 0.6315192743764172,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.007679071961320005,
|
|
"learning_rate": 3.6904761904761906e-07,
|
|
"loss": 0.016,
|
|
"num_tokens": 65888302.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1114,
|
|
"step_time": 52.48979418724775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1321.0,
|
|
"completions/max_terminated_length": 1321.0,
|
|
"completions/mean_length": 751.78125,
|
|
"completions/mean_terminated_length": 751.78125,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.9594408832490444,
|
|
"epoch": 0.6320861678004536,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.0075347155798226595,
|
|
"learning_rate": 3.6848072562358275e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 65946728.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1115,
|
|
"step_time": 46.334358433261514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1361.0,
|
|
"completions/max_terminated_length": 1361.0,
|
|
"completions/mean_length": 760.984375,
|
|
"completions/mean_terminated_length": 760.984375,
|
|
"completions/min_length": 188.0,
|
|
"completions/min_terminated_length": 188.0,
|
|
"entropy": 0.7617907468229532,
|
|
"epoch": 0.6326530612244898,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.006829036719864234,
|
|
"learning_rate": 3.6791383219954645e-07,
|
|
"loss": 0.0339,
|
|
"num_tokens": 66004215.0,
|
|
"reward": 0.46875,
|
|
"reward_std": 0.7961963415145874,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1116,
|
|
"step_time": 47.501841524615884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1396.0,
|
|
"completions/max_terminated_length": 1396.0,
|
|
"completions/mean_length": 792.03125,
|
|
"completions/mean_terminated_length": 792.03125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.8017204832285643,
|
|
"epoch": 0.6332199546485261,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.005459077510749921,
|
|
"learning_rate": 3.673469387755102e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 66066313.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1117,
|
|
"step_time": 50.13531224709004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1267.0,
|
|
"completions/max_terminated_length": 1267.0,
|
|
"completions/mean_length": 846.734375,
|
|
"completions/mean_terminated_length": 846.734375,
|
|
"completions/min_length": 444.0,
|
|
"completions/min_terminated_length": 444.0,
|
|
"entropy": 0.804381500929594,
|
|
"epoch": 0.6337868480725624,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.006022864254191518,
|
|
"learning_rate": 3.667800453514739e-07,
|
|
"loss": 0.0192,
|
|
"num_tokens": 66130144.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1118,
|
|
"step_time": 44.469872353598475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2109.0,
|
|
"completions/max_terminated_length": 2109.0,
|
|
"completions/mean_length": 714.140625,
|
|
"completions/mean_terminated_length": 714.140625,
|
|
"completions/min_length": 229.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"entropy": 0.8198220022022724,
|
|
"epoch": 0.6343537414965986,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.012241557080415078,
|
|
"learning_rate": 3.6621315192743765e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 66188409.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6871842741966248,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1119,
|
|
"step_time": 78.07691939454526
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1238.0,
|
|
"completions/max_terminated_length": 1238.0,
|
|
"completions/mean_length": 828.0625,
|
|
"completions/mean_terminated_length": 828.0625,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.7281496822834015,
|
|
"epoch": 0.6349206349206349,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.007427489006659016,
|
|
"learning_rate": 3.6564625850340134e-07,
|
|
"loss": 0.0296,
|
|
"num_tokens": 66252173.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1120,
|
|
"step_time": 43.919082765467465
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1201.0,
|
|
"completions/max_terminated_length": 1201.0,
|
|
"completions/mean_length": 659.109375,
|
|
"completions/mean_terminated_length": 659.109375,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.7508130557835102,
|
|
"epoch": 0.6354875283446711,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.008689042093465105,
|
|
"learning_rate": 3.6507936507936504e-07,
|
|
"loss": 0.0081,
|
|
"num_tokens": 66303644.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.7332369089126587,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.1015625,
|
|
"rewards/format_reward_func/std": 0.20275264978408813,
|
|
"step": 1121,
|
|
"step_time": 41.70680932048708
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/max_terminated_length": 1376.0,
|
|
"completions/mean_length": 627.796875,
|
|
"completions/mean_terminated_length": 627.796875,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.811627546325326,
|
|
"epoch": 0.6360544217687075,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.65234375,
|
|
"kl": 0.015093438094481826,
|
|
"learning_rate": 3.645124716553288e-07,
|
|
"loss": -0.0334,
|
|
"num_tokens": 66356215.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1122,
|
|
"step_time": 49.52775328513235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1300.0,
|
|
"completions/max_terminated_length": 1300.0,
|
|
"completions/mean_length": 771.734375,
|
|
"completions/mean_terminated_length": 771.734375,
|
|
"completions/min_length": 352.0,
|
|
"completions/min_terminated_length": 352.0,
|
|
"entropy": 0.7302479594945908,
|
|
"epoch": 0.6366213151927438,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.287109375,
|
|
"kl": 0.008716506490600295,
|
|
"learning_rate": 3.6394557823129254e-07,
|
|
"loss": 0.0146,
|
|
"num_tokens": 66415854.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1123,
|
|
"step_time": 45.50562099646777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1278.0,
|
|
"completions/max_terminated_length": 1278.0,
|
|
"completions/mean_length": 751.1875,
|
|
"completions/mean_terminated_length": 751.1875,
|
|
"completions/min_length": 340.0,
|
|
"completions/min_terminated_length": 340.0,
|
|
"entropy": 0.8541635610163212,
|
|
"epoch": 0.63718820861678,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.015641508536646143,
|
|
"learning_rate": 3.633786848072562e-07,
|
|
"loss": -0.0065,
|
|
"num_tokens": 66475330.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6782841682434082,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1124,
|
|
"step_time": 45.261970506981015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1233.0,
|
|
"completions/mean_length": 765.21875,
|
|
"completions/mean_terminated_length": 712.3492431640625,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 1.0287728533148766,
|
|
"epoch": 0.6377551020408163,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.00937877141404897,
|
|
"learning_rate": 3.6281179138321993e-07,
|
|
"loss": -0.0231,
|
|
"num_tokens": 66535328.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1125,
|
|
"step_time": 173.93311240151525
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1619.0,
|
|
"completions/max_terminated_length": 1619.0,
|
|
"completions/mean_length": 831.9375,
|
|
"completions/mean_terminated_length": 831.9375,
|
|
"completions/min_length": 288.0,
|
|
"completions/min_terminated_length": 288.0,
|
|
"entropy": 0.6449856609106064,
|
|
"epoch": 0.6383219954648526,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.008782715376582928,
|
|
"learning_rate": 3.622448979591837e-07,
|
|
"loss": 0.0005,
|
|
"num_tokens": 66599100.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.6950790882110596,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1126,
|
|
"step_time": 57.564347558654845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1140.0,
|
|
"completions/max_terminated_length": 1140.0,
|
|
"completions/mean_length": 695.265625,
|
|
"completions/mean_terminated_length": 695.265625,
|
|
"completions/min_length": 286.0,
|
|
"completions/min_terminated_length": 286.0,
|
|
"entropy": 0.9345854353159666,
|
|
"epoch": 0.6388888888888888,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.013184856215957552,
|
|
"learning_rate": 3.616780045351474e-07,
|
|
"loss": 0.0164,
|
|
"num_tokens": 66654565.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1127,
|
|
"step_time": 40.558634859509766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1555.0,
|
|
"completions/max_terminated_length": 1555.0,
|
|
"completions/mean_length": 793.8125,
|
|
"completions/mean_terminated_length": 793.8125,
|
|
"completions/min_length": 242.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"entropy": 0.738713588565588,
|
|
"epoch": 0.6394557823129252,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.008480313335894607,
|
|
"learning_rate": 3.6111111111111107e-07,
|
|
"loss": -0.007,
|
|
"num_tokens": 66714937.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1128,
|
|
"step_time": 54.4101136745885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1240.0,
|
|
"completions/max_terminated_length": 1240.0,
|
|
"completions/mean_length": 725.921875,
|
|
"completions/mean_terminated_length": 725.921875,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.8166520409286022,
|
|
"epoch": 0.6400226757369615,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.010748002823675051,
|
|
"learning_rate": 3.605442176870748e-07,
|
|
"loss": 0.026,
|
|
"num_tokens": 66772732.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1129,
|
|
"step_time": 43.7646795893088
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1139.0,
|
|
"completions/max_terminated_length": 1139.0,
|
|
"completions/mean_length": 724.09375,
|
|
"completions/mean_terminated_length": 724.09375,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.807806920260191,
|
|
"epoch": 0.6405895691609977,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.01643386864452623,
|
|
"learning_rate": 3.599773242630385e-07,
|
|
"loss": -0.0244,
|
|
"num_tokens": 66829698.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6871842741966248,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1130,
|
|
"step_time": 40.39699596352875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1681.0,
|
|
"completions/max_terminated_length": 1681.0,
|
|
"completions/mean_length": 741.078125,
|
|
"completions/mean_terminated_length": 741.078125,
|
|
"completions/min_length": 304.0,
|
|
"completions/min_terminated_length": 304.0,
|
|
"entropy": 1.0368301719427109,
|
|
"epoch": 0.641156462585034,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"kl": 0.008527922007488087,
|
|
"learning_rate": 3.5941043083900227e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 66891791.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1131,
|
|
"step_time": 61.42025569733232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1546.0,
|
|
"completions/max_terminated_length": 1546.0,
|
|
"completions/mean_length": 783.21875,
|
|
"completions/mean_terminated_length": 783.21875,
|
|
"completions/min_length": 224.0,
|
|
"completions/min_terminated_length": 224.0,
|
|
"entropy": 0.8265883233398199,
|
|
"epoch": 0.6417233560090703,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.014500535558909178,
|
|
"learning_rate": 3.5884353741496596e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 66955173.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1132,
|
|
"step_time": 56.99815191142261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1600.0,
|
|
"completions/max_terminated_length": 1600.0,
|
|
"completions/mean_length": 886.59375,
|
|
"completions/mean_terminated_length": 886.59375,
|
|
"completions/min_length": 434.0,
|
|
"completions/min_terminated_length": 434.0,
|
|
"entropy": 0.7918216288089752,
|
|
"epoch": 0.6422902494331065,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.482421875,
|
|
"kl": 0.008505289268214256,
|
|
"learning_rate": 3.582766439909297e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 67021627.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1133,
|
|
"step_time": 57.82349129579961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1341.0,
|
|
"completions/max_terminated_length": 1341.0,
|
|
"completions/mean_length": 757.109375,
|
|
"completions/mean_terminated_length": 757.109375,
|
|
"completions/min_length": 404.0,
|
|
"completions/min_terminated_length": 404.0,
|
|
"entropy": 0.922251746058464,
|
|
"epoch": 0.6428571428571429,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.007271447786479257,
|
|
"learning_rate": 3.577097505668934e-07,
|
|
"loss": -0.024,
|
|
"num_tokens": 67081314.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7820135354995728,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1134,
|
|
"step_time": 46.98732428532094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1311.0,
|
|
"completions/max_terminated_length": 1311.0,
|
|
"completions/mean_length": 720.90625,
|
|
"completions/mean_terminated_length": 720.90625,
|
|
"completions/min_length": 172.0,
|
|
"completions/min_terminated_length": 172.0,
|
|
"entropy": 0.7367459032684565,
|
|
"epoch": 0.6434240362811792,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.007066757214488462,
|
|
"learning_rate": 3.5714285714285716e-07,
|
|
"loss": -0.0204,
|
|
"num_tokens": 67135572.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1135,
|
|
"step_time": 45.59942451491952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1846.0,
|
|
"completions/max_terminated_length": 1846.0,
|
|
"completions/mean_length": 837.578125,
|
|
"completions/mean_terminated_length": 837.578125,
|
|
"completions/min_length": 420.0,
|
|
"completions/min_terminated_length": 420.0,
|
|
"entropy": 0.7234981674700975,
|
|
"epoch": 0.6439909297052154,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.008606898714788258,
|
|
"learning_rate": 3.5657596371882085e-07,
|
|
"loss": -0.002,
|
|
"num_tokens": 67200545.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1136,
|
|
"step_time": 65.81486516725272
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1296.0,
|
|
"completions/max_terminated_length": 1296.0,
|
|
"completions/mean_length": 708.390625,
|
|
"completions/mean_terminated_length": 708.390625,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.9345072396099567,
|
|
"epoch": 0.6445578231292517,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.333984375,
|
|
"kl": 0.006655256365775131,
|
|
"learning_rate": 3.5600907029478455e-07,
|
|
"loss": 0.0054,
|
|
"num_tokens": 67256482.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1137,
|
|
"step_time": 45.47106663696468
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1277.0,
|
|
"completions/max_terminated_length": 1277.0,
|
|
"completions/mean_length": 705.390625,
|
|
"completions/mean_terminated_length": 705.390625,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.9238828830420971,
|
|
"epoch": 0.645124716553288,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.007699513036641292,
|
|
"learning_rate": 3.554421768707483e-07,
|
|
"loss": 0.0005,
|
|
"num_tokens": 67313707.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1138,
|
|
"step_time": 45.25024778302759
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1616.0,
|
|
"completions/max_terminated_length": 1616.0,
|
|
"completions/mean_length": 847.390625,
|
|
"completions/mean_terminated_length": 847.390625,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7924791537225246,
|
|
"epoch": 0.6456916099773242,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.011052212212234735,
|
|
"learning_rate": 3.5487528344671205e-07,
|
|
"loss": 0.0115,
|
|
"num_tokens": 67379244.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 1139,
|
|
"step_time": 56.85620425734669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2372.0,
|
|
"completions/max_terminated_length": 2372.0,
|
|
"completions/mean_length": 861.03125,
|
|
"completions/mean_terminated_length": 861.03125,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.6256184093654156,
|
|
"epoch": 0.6462585034013606,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.431640625,
|
|
"kl": 0.00483911664923653,
|
|
"learning_rate": 3.543083900226757e-07,
|
|
"loss": -0.0337,
|
|
"num_tokens": 67443934.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.3117053508758545,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.1015625,
|
|
"rewards/format_reward_func/std": 0.20275264978408813,
|
|
"step": 1140,
|
|
"step_time": 85.63249838165939
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1670.0,
|
|
"completions/max_terminated_length": 1670.0,
|
|
"completions/mean_length": 747.203125,
|
|
"completions/mean_terminated_length": 747.203125,
|
|
"completions/min_length": 178.0,
|
|
"completions/min_terminated_length": 178.0,
|
|
"entropy": 0.6919434871524572,
|
|
"epoch": 0.6468253968253969,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.42578125,
|
|
"kl": 0.013656156545039266,
|
|
"learning_rate": 3.5374149659863944e-07,
|
|
"loss": 0.014,
|
|
"num_tokens": 67503739.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1141,
|
|
"step_time": 59.901625351049006
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1732.0,
|
|
"completions/max_terminated_length": 1732.0,
|
|
"completions/mean_length": 803.34375,
|
|
"completions/mean_terminated_length": 803.34375,
|
|
"completions/min_length": 365.0,
|
|
"completions/min_terminated_length": 365.0,
|
|
"entropy": 0.7998859602957964,
|
|
"epoch": 0.6473922902494331,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.010639237734721974,
|
|
"learning_rate": 3.531746031746032e-07,
|
|
"loss": 0.0143,
|
|
"num_tokens": 67565457.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1142,
|
|
"step_time": 61.434579437598586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1056.0,
|
|
"completions/mean_length": 659.28125,
|
|
"completions/mean_terminated_length": 604.7301635742188,
|
|
"completions/min_length": 326.0,
|
|
"completions/min_terminated_length": 326.0,
|
|
"entropy": 0.8404234498739243,
|
|
"epoch": 0.6479591836734694,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.74609375,
|
|
"kl": 0.011651739740045741,
|
|
"learning_rate": 3.526077097505669e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 67617379.0,
|
|
"reward": 0.53125,
|
|
"reward_std": 0.8539125919342041,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1143,
|
|
"step_time": 171.54008629452437
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2703.0,
|
|
"completions/max_terminated_length": 2703.0,
|
|
"completions/mean_length": 708.5625,
|
|
"completions/mean_terminated_length": 708.5625,
|
|
"completions/min_length": 295.0,
|
|
"completions/min_terminated_length": 295.0,
|
|
"entropy": 0.6106820367276669,
|
|
"epoch": 0.6485260770975056,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"kl": 0.007240210165036842,
|
|
"learning_rate": 3.520408163265306e-07,
|
|
"loss": -0.0202,
|
|
"num_tokens": 67672007.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7663560509681702,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1144,
|
|
"step_time": 100.74209997802973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2022.0,
|
|
"completions/max_terminated_length": 2022.0,
|
|
"completions/mean_length": 723.5,
|
|
"completions/mean_terminated_length": 723.5,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.7202003058046103,
|
|
"epoch": 0.6490929705215419,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6484375,
|
|
"kl": 0.024503982975147665,
|
|
"learning_rate": 3.5147392290249433e-07,
|
|
"loss": -0.0421,
|
|
"num_tokens": 67729967.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1145,
|
|
"step_time": 72.29018565453589
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2031.0,
|
|
"completions/max_terminated_length": 2031.0,
|
|
"completions/mean_length": 750.546875,
|
|
"completions/mean_terminated_length": 750.546875,
|
|
"completions/min_length": 192.0,
|
|
"completions/min_terminated_length": 192.0,
|
|
"entropy": 0.6171744558960199,
|
|
"epoch": 0.6496598639455783,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.014786920393817127,
|
|
"learning_rate": 3.5090702947845803e-07,
|
|
"loss": -0.0422,
|
|
"num_tokens": 67789002.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.7325600981712341,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1146,
|
|
"step_time": 73.6035605892539
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1651.0,
|
|
"completions/max_terminated_length": 1651.0,
|
|
"completions/mean_length": 782.359375,
|
|
"completions/mean_terminated_length": 782.359375,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 1.0600887797772884,
|
|
"epoch": 0.6502267573696145,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33984375,
|
|
"kl": 0.005793417556560598,
|
|
"learning_rate": 3.503401360544218e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 67849937.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.417855441570282,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1147,
|
|
"step_time": 58.40652887150645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1272.0,
|
|
"completions/max_terminated_length": 1272.0,
|
|
"completions/mean_length": 728.125,
|
|
"completions/mean_terminated_length": 728.125,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.7972042448818684,
|
|
"epoch": 0.6507936507936508,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.019399877841351554,
|
|
"learning_rate": 3.4977324263038547e-07,
|
|
"loss": 0.0309,
|
|
"num_tokens": 67906185.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1148,
|
|
"step_time": 45.57844638545066
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1366.0,
|
|
"completions/max_terminated_length": 1366.0,
|
|
"completions/mean_length": 668.921875,
|
|
"completions/mean_terminated_length": 668.921875,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.9734818674623966,
|
|
"epoch": 0.6513605442176871,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.01762492602574639,
|
|
"learning_rate": 3.4920634920634917e-07,
|
|
"loss": 0.0141,
|
|
"num_tokens": 67960540.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6538955569267273,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1149,
|
|
"step_time": 48.641934112645686
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1324.0,
|
|
"completions/max_terminated_length": 1324.0,
|
|
"completions/mean_length": 779.046875,
|
|
"completions/mean_terminated_length": 779.046875,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.8373247776180506,
|
|
"epoch": 0.6519274376417233,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.019896175537724048,
|
|
"learning_rate": 3.486394557823129e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 68022335.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1150,
|
|
"step_time": 47.64535669237375
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1740.0,
|
|
"completions/max_terminated_length": 1740.0,
|
|
"completions/mean_length": 805.25,
|
|
"completions/mean_terminated_length": 805.25,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.6873206682503223,
|
|
"epoch": 0.6524943310657596,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.0068439550523180515,
|
|
"learning_rate": 3.4807256235827667e-07,
|
|
"loss": -0.0067,
|
|
"num_tokens": 68082431.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6878606677055359,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1151,
|
|
"step_time": 61.63420953415334
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1467.0,
|
|
"completions/max_terminated_length": 1467.0,
|
|
"completions/mean_length": 765.96875,
|
|
"completions/mean_terminated_length": 765.96875,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.6675137281417847,
|
|
"epoch": 0.6530612244897959,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.008595615057856776,
|
|
"learning_rate": 3.475056689342403e-07,
|
|
"loss": 0.0063,
|
|
"num_tokens": 68140757.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1152,
|
|
"step_time": 50.947300830855966
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1312.0,
|
|
"completions/max_terminated_length": 1312.0,
|
|
"completions/mean_length": 798.046875,
|
|
"completions/mean_terminated_length": 798.046875,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.8677041567862034,
|
|
"epoch": 0.6536281179138322,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3203125,
|
|
"kl": 0.007954949702252634,
|
|
"learning_rate": 3.4693877551020406e-07,
|
|
"loss": 0.0218,
|
|
"num_tokens": 68202992.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1153,
|
|
"step_time": 47.17798578646034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1245.0,
|
|
"completions/max_terminated_length": 1245.0,
|
|
"completions/mean_length": 787.625,
|
|
"completions/mean_terminated_length": 787.625,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.7481411583721638,
|
|
"epoch": 0.6541950113378685,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.006574676546733826,
|
|
"learning_rate": 3.463718820861678e-07,
|
|
"loss": 0.0119,
|
|
"num_tokens": 68263264.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1154,
|
|
"step_time": 43.70536099001765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1159.0,
|
|
"completions/max_terminated_length": 1159.0,
|
|
"completions/mean_length": 712.359375,
|
|
"completions/mean_terminated_length": 712.359375,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.9394162930548191,
|
|
"epoch": 0.6547619047619048,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2470703125,
|
|
"kl": 0.008892172307241708,
|
|
"learning_rate": 3.458049886621315e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 68319383.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1155,
|
|
"step_time": 41.3480468345806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1493.0,
|
|
"completions/mean_length": 783.4375,
|
|
"completions/mean_terminated_length": 730.857177734375,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.8277614675462246,
|
|
"epoch": 0.655328798185941,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.010530049752560444,
|
|
"learning_rate": 3.452380952380952e-07,
|
|
"loss": -0.0037,
|
|
"num_tokens": 68381803.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1156,
|
|
"step_time": 174.7410236382857
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1481.0,
|
|
"completions/max_terminated_length": 1481.0,
|
|
"completions/mean_length": 737.90625,
|
|
"completions/mean_terminated_length": 737.90625,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 0.7657451219856739,
|
|
"epoch": 0.6558956916099773,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.009367741382448003,
|
|
"learning_rate": 3.4467120181405895e-07,
|
|
"loss": -0.0245,
|
|
"num_tokens": 68438733.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.8886408805847168,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1157,
|
|
"step_time": 52.45601293258369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1310.0,
|
|
"completions/max_terminated_length": 1310.0,
|
|
"completions/mean_length": 770.640625,
|
|
"completions/mean_terminated_length": 770.640625,
|
|
"completions/min_length": 333.0,
|
|
"completions/min_terminated_length": 333.0,
|
|
"entropy": 0.767628775909543,
|
|
"epoch": 0.6564625850340136,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.006680283389869146,
|
|
"learning_rate": 3.4410430839002265e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 68497094.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1158,
|
|
"step_time": 46.18379644304514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2186.0,
|
|
"completions/max_terminated_length": 2186.0,
|
|
"completions/mean_length": 797.703125,
|
|
"completions/mean_terminated_length": 797.703125,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.8094903975725174,
|
|
"epoch": 0.6570294784580499,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"kl": 0.008846655153320171,
|
|
"learning_rate": 3.435374149659864e-07,
|
|
"loss": 0.0373,
|
|
"num_tokens": 68558003.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1159,
|
|
"step_time": 78.62068361509591
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 661.5625,
|
|
"completions/mean_terminated_length": 661.5625,
|
|
"completions/min_length": 248.0,
|
|
"completions/min_terminated_length": 248.0,
|
|
"entropy": 0.6736507806926966,
|
|
"epoch": 0.6575963718820862,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.00915847605210729,
|
|
"learning_rate": 3.429705215419501e-07,
|
|
"loss": -0.0202,
|
|
"num_tokens": 68612255.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1160,
|
|
"step_time": 46.68086963985115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1430.0,
|
|
"completions/max_terminated_length": 1430.0,
|
|
"completions/mean_length": 748.53125,
|
|
"completions/mean_terminated_length": 748.53125,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.8270637579262257,
|
|
"epoch": 0.6581632653061225,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.01122271767235361,
|
|
"learning_rate": 3.424036281179138e-07,
|
|
"loss": -0.005,
|
|
"num_tokens": 68669657.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6423677802085876,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1161,
|
|
"step_time": 50.34987534210086
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1445.0,
|
|
"completions/max_terminated_length": 1445.0,
|
|
"completions/mean_length": 763.046875,
|
|
"completions/mean_terminated_length": 763.046875,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.8949731532484293,
|
|
"epoch": 0.6587301587301587,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6171875,
|
|
"kl": 0.00571693285019137,
|
|
"learning_rate": 3.4183673469387754e-07,
|
|
"loss": 0.01,
|
|
"num_tokens": 68728892.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8003905415534973,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1162,
|
|
"step_time": 51.49605161976069
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1443.0,
|
|
"completions/max_terminated_length": 1443.0,
|
|
"completions/mean_length": 794.015625,
|
|
"completions/mean_terminated_length": 794.015625,
|
|
"completions/min_length": 212.0,
|
|
"completions/min_terminated_length": 212.0,
|
|
"entropy": 0.8076041843742132,
|
|
"epoch": 0.659297052154195,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.007915709400549531,
|
|
"learning_rate": 3.412698412698413e-07,
|
|
"loss": -0.0172,
|
|
"num_tokens": 68793477.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1163,
|
|
"step_time": 52.672715849243104
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/max_terminated_length": 1452.0,
|
|
"completions/mean_length": 877.65625,
|
|
"completions/mean_terminated_length": 877.65625,
|
|
"completions/min_length": 230.0,
|
|
"completions/min_terminated_length": 230.0,
|
|
"entropy": 0.6329321265220642,
|
|
"epoch": 0.6598639455782312,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.00734649394871667,
|
|
"learning_rate": 3.40702947845805e-07,
|
|
"loss": -0.0219,
|
|
"num_tokens": 68860575.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1164,
|
|
"step_time": 51.8628015993163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1282.0,
|
|
"completions/max_terminated_length": 1282.0,
|
|
"completions/mean_length": 686.5,
|
|
"completions/mean_terminated_length": 686.5,
|
|
"completions/min_length": 248.0,
|
|
"completions/min_terminated_length": 248.0,
|
|
"entropy": 0.7428093068301678,
|
|
"epoch": 0.6604308390022676,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.006841927155619487,
|
|
"learning_rate": 3.401360544217687e-07,
|
|
"loss": -0.0223,
|
|
"num_tokens": 68915095.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1165,
|
|
"step_time": 45.379655653610826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3764.0,
|
|
"completions/max_terminated_length": 3764.0,
|
|
"completions/mean_length": 1029.59375,
|
|
"completions/mean_terminated_length": 1029.59375,
|
|
"completions/min_length": 541.0,
|
|
"completions/min_terminated_length": 541.0,
|
|
"entropy": 0.5289673125371337,
|
|
"epoch": 0.6609977324263039,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.451171875,
|
|
"kl": 0.0061146316438680515,
|
|
"learning_rate": 3.3956916099773243e-07,
|
|
"loss": -0.0067,
|
|
"num_tokens": 68992437.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.835710883140564,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1166,
|
|
"step_time": 154.97246967442334
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1136.0,
|
|
"completions/max_terminated_length": 1136.0,
|
|
"completions/mean_length": 682.515625,
|
|
"completions/mean_terminated_length": 682.515625,
|
|
"completions/min_length": 347.0,
|
|
"completions/min_terminated_length": 347.0,
|
|
"entropy": 1.0174338072538376,
|
|
"epoch": 0.6615646258503401,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.76953125,
|
|
"kl": 0.009873627801425755,
|
|
"learning_rate": 3.390022675736962e-07,
|
|
"loss": -0.0269,
|
|
"num_tokens": 69045462.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1167,
|
|
"step_time": 40.69202217459679
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1394.0,
|
|
"completions/max_terminated_length": 1394.0,
|
|
"completions/mean_length": 783.0625,
|
|
"completions/mean_terminated_length": 783.0625,
|
|
"completions/min_length": 294.0,
|
|
"completions/min_terminated_length": 294.0,
|
|
"entropy": 0.8566990979015827,
|
|
"epoch": 0.6621315192743764,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.02251550421351567,
|
|
"learning_rate": 3.384353741496598e-07,
|
|
"loss": -0.0028,
|
|
"num_tokens": 69106386.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.7539166808128357,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1168,
|
|
"step_time": 48.97253660019487
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1307.0,
|
|
"completions/max_terminated_length": 1307.0,
|
|
"completions/mean_length": 814.15625,
|
|
"completions/mean_terminated_length": 814.15625,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.7652460020035505,
|
|
"epoch": 0.6626984126984127,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.016276433918392286,
|
|
"learning_rate": 3.3786848072562357e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 69168788.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1169,
|
|
"step_time": 46.17171176336706
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1257.0,
|
|
"completions/max_terminated_length": 1257.0,
|
|
"completions/mean_length": 718.375,
|
|
"completions/mean_terminated_length": 718.375,
|
|
"completions/min_length": 300.0,
|
|
"completions/min_terminated_length": 300.0,
|
|
"entropy": 0.8557120319455862,
|
|
"epoch": 0.6632653061224489,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.007913934183306992,
|
|
"learning_rate": 3.373015873015873e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 69229476.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.8858454823493958,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1170,
|
|
"step_time": 45.45499105844647
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1116.0,
|
|
"completions/max_terminated_length": 1116.0,
|
|
"completions/mean_length": 729.765625,
|
|
"completions/mean_terminated_length": 729.765625,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7916884180158377,
|
|
"epoch": 0.6638321995464853,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.39453125,
|
|
"kl": 0.006144991508335806,
|
|
"learning_rate": 3.3673469387755096e-07,
|
|
"loss": 0.0025,
|
|
"num_tokens": 69285789.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1171,
|
|
"step_time": 39.34412661101669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2064.0,
|
|
"completions/max_terminated_length": 2064.0,
|
|
"completions/mean_length": 837.546875,
|
|
"completions/mean_terminated_length": 837.546875,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.6938365995883942,
|
|
"epoch": 0.6643990929705216,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.010861321687116288,
|
|
"learning_rate": 3.361678004535147e-07,
|
|
"loss": -0.0132,
|
|
"num_tokens": 69351984.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1172,
|
|
"step_time": 74.32535912748426
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1280.0,
|
|
"completions/max_terminated_length": 1280.0,
|
|
"completions/mean_length": 774.40625,
|
|
"completions/mean_terminated_length": 774.40625,
|
|
"completions/min_length": 361.0,
|
|
"completions/min_terminated_length": 361.0,
|
|
"entropy": 0.830631460994482,
|
|
"epoch": 0.6649659863945578,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.009048327076015994,
|
|
"learning_rate": 3.3560090702947846e-07,
|
|
"loss": -0.0268,
|
|
"num_tokens": 69414114.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1173,
|
|
"step_time": 45.74139649514109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1372.0,
|
|
"completions/max_terminated_length": 1372.0,
|
|
"completions/mean_length": 776.453125,
|
|
"completions/mean_terminated_length": 776.453125,
|
|
"completions/min_length": 117.0,
|
|
"completions/min_terminated_length": 117.0,
|
|
"entropy": 0.7854088321328163,
|
|
"epoch": 0.6655328798185941,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.008421445090789348,
|
|
"learning_rate": 3.3503401360544216e-07,
|
|
"loss": -0.035,
|
|
"num_tokens": 69475007.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1174,
|
|
"step_time": 49.16447034385055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1156.0,
|
|
"completions/max_terminated_length": 1156.0,
|
|
"completions/mean_length": 647.59375,
|
|
"completions/mean_terminated_length": 647.59375,
|
|
"completions/min_length": 180.0,
|
|
"completions/min_terminated_length": 180.0,
|
|
"entropy": 1.001741111278534,
|
|
"epoch": 0.6660997732426304,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.62109375,
|
|
"kl": 0.008010145218577236,
|
|
"learning_rate": 3.344671201814059e-07,
|
|
"loss": -0.0026,
|
|
"num_tokens": 69526325.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1175,
|
|
"step_time": 40.75483793951571
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1475.0,
|
|
"completions/max_terminated_length": 1475.0,
|
|
"completions/mean_length": 859.375,
|
|
"completions/mean_terminated_length": 859.375,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.7295182626694441,
|
|
"epoch": 0.6666666666666666,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.498046875,
|
|
"kl": 0.005891725028050132,
|
|
"learning_rate": 3.339002267573696e-07,
|
|
"loss": 0.0497,
|
|
"num_tokens": 69595437.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1176,
|
|
"step_time": 55.39253865275532
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1365.0,
|
|
"completions/max_terminated_length": 1365.0,
|
|
"completions/mean_length": 596.4375,
|
|
"completions/mean_terminated_length": 596.4375,
|
|
"completions/min_length": 200.0,
|
|
"completions/min_terminated_length": 200.0,
|
|
"entropy": 0.6896672677248716,
|
|
"epoch": 0.6672335600907029,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6953125,
|
|
"kl": 0.014278341521276161,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"loss": -0.0343,
|
|
"num_tokens": 69642609.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.774084210395813,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1177,
|
|
"step_time": 48.37230580300093
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1431.0,
|
|
"completions/max_terminated_length": 1431.0,
|
|
"completions/mean_length": 704.125,
|
|
"completions/mean_terminated_length": 704.125,
|
|
"completions/min_length": 154.0,
|
|
"completions/min_terminated_length": 154.0,
|
|
"entropy": 0.9396607503294945,
|
|
"epoch": 0.6678004535147393,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7734375,
|
|
"kl": 0.011146384102175944,
|
|
"learning_rate": 3.3276643990929705e-07,
|
|
"loss": -0.0419,
|
|
"num_tokens": 69698089.0,
|
|
"reward": 0.421875,
|
|
"reward_std": 0.7777600884437561,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1178,
|
|
"step_time": 51.16614926978946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1327.0,
|
|
"completions/max_terminated_length": 1327.0,
|
|
"completions/mean_length": 696.328125,
|
|
"completions/mean_terminated_length": 696.328125,
|
|
"completions/min_length": 351.0,
|
|
"completions/min_terminated_length": 351.0,
|
|
"entropy": 0.6481283120810986,
|
|
"epoch": 0.6683673469387755,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66796875,
|
|
"kl": 0.020628180966014042,
|
|
"learning_rate": 3.321995464852608e-07,
|
|
"loss": 0.0261,
|
|
"num_tokens": 69755350.0,
|
|
"reward": 0.65625,
|
|
"reward_std": 0.8947598934173584,
|
|
"rewards/accuracy_reward_func/mean": 0.59375,
|
|
"rewards/accuracy_reward_func/std": 0.9209855198860168,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1179,
|
|
"step_time": 47.86243663635105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1171.0,
|
|
"completions/max_terminated_length": 1171.0,
|
|
"completions/mean_length": 708.59375,
|
|
"completions/mean_terminated_length": 708.59375,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.7681530825793743,
|
|
"epoch": 0.6689342403628118,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59765625,
|
|
"kl": 0.008725753017643001,
|
|
"learning_rate": 3.3163265306122444e-07,
|
|
"loss": 0.0294,
|
|
"num_tokens": 69811396.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1180,
|
|
"step_time": 42.019110477529466
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1247.0,
|
|
"completions/max_terminated_length": 1247.0,
|
|
"completions/mean_length": 772.171875,
|
|
"completions/mean_terminated_length": 772.171875,
|
|
"completions/min_length": 429.0,
|
|
"completions/min_terminated_length": 429.0,
|
|
"entropy": 0.6991284471005201,
|
|
"epoch": 0.6695011337868481,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.010233829962089658,
|
|
"learning_rate": 3.310657596371882e-07,
|
|
"loss": 0.02,
|
|
"num_tokens": 69873711.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.7116082906723022,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1181,
|
|
"step_time": 44.49350086040795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1440.0,
|
|
"completions/max_terminated_length": 1440.0,
|
|
"completions/mean_length": 759.75,
|
|
"completions/mean_terminated_length": 759.75,
|
|
"completions/min_length": 373.0,
|
|
"completions/min_terminated_length": 373.0,
|
|
"entropy": 0.7347757127135992,
|
|
"epoch": 0.6700680272108843,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.0130808693356812,
|
|
"learning_rate": 3.3049886621315194e-07,
|
|
"loss": 0.0237,
|
|
"num_tokens": 69933791.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 1182,
|
|
"step_time": 51.56232512369752
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1403.0,
|
|
"completions/max_terminated_length": 1403.0,
|
|
"completions/mean_length": 749.265625,
|
|
"completions/mean_terminated_length": 749.265625,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.9692546874284744,
|
|
"epoch": 0.6706349206349206,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.7578125,
|
|
"kl": 0.010821497213328257,
|
|
"learning_rate": 3.299319727891156e-07,
|
|
"loss": -0.0214,
|
|
"num_tokens": 69994040.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1183,
|
|
"step_time": 50.31179391220212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1558.0,
|
|
"completions/max_terminated_length": 1558.0,
|
|
"completions/mean_length": 726.421875,
|
|
"completions/mean_terminated_length": 726.421875,
|
|
"completions/min_length": 209.0,
|
|
"completions/min_terminated_length": 209.0,
|
|
"entropy": 0.7688326518982649,
|
|
"epoch": 0.671201814058957,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.012261275202035904,
|
|
"learning_rate": 3.2936507936507933e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 70051499.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8188852071762085,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1184,
|
|
"step_time": 54.836858629249036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1682.0,
|
|
"completions/max_terminated_length": 1682.0,
|
|
"completions/mean_length": 782.90625,
|
|
"completions/mean_terminated_length": 782.90625,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.9850017391145229,
|
|
"epoch": 0.6717687074829932,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.010726811015047133,
|
|
"learning_rate": 3.287981859410431e-07,
|
|
"loss": -0.0006,
|
|
"num_tokens": 70112173.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5747131109237671,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1185,
|
|
"step_time": 59.472431069239974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2194.0,
|
|
"completions/max_terminated_length": 2194.0,
|
|
"completions/mean_length": 801.078125,
|
|
"completions/mean_terminated_length": 801.078125,
|
|
"completions/min_length": 187.0,
|
|
"completions/min_terminated_length": 187.0,
|
|
"entropy": 0.9212711304426193,
|
|
"epoch": 0.6723356009070295,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41015625,
|
|
"kl": 0.011708885736879893,
|
|
"learning_rate": 3.282312925170068e-07,
|
|
"loss": -0.0049,
|
|
"num_tokens": 70174002.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1186,
|
|
"step_time": 78.9440293777734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1276.0,
|
|
"completions/max_terminated_length": 1276.0,
|
|
"completions/mean_length": 756.0,
|
|
"completions/mean_terminated_length": 756.0,
|
|
"completions/min_length": 119.0,
|
|
"completions/min_terminated_length": 119.0,
|
|
"entropy": 0.9305983707308769,
|
|
"epoch": 0.6729024943310657,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.0086804735037731,
|
|
"learning_rate": 3.276643990929705e-07,
|
|
"loss": 0.015,
|
|
"num_tokens": 70231906.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1187,
|
|
"step_time": 45.16399548947811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1483.0,
|
|
"completions/max_terminated_length": 1483.0,
|
|
"completions/mean_length": 708.875,
|
|
"completions/mean_terminated_length": 708.875,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.7685298956930637,
|
|
"epoch": 0.673469387755102,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3984375,
|
|
"kl": 0.01945055641408544,
|
|
"learning_rate": 3.270975056689342e-07,
|
|
"loss": -0.0021,
|
|
"num_tokens": 70288930.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6777355074882507,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1188,
|
|
"step_time": 52.87077456340194
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1423.0,
|
|
"completions/max_terminated_length": 1423.0,
|
|
"completions/mean_length": 760.96875,
|
|
"completions/mean_terminated_length": 760.96875,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 1.0578239634633064,
|
|
"epoch": 0.6740362811791383,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.400390625,
|
|
"kl": 0.010812026623170823,
|
|
"learning_rate": 3.265306122448979e-07,
|
|
"loss": 0.0175,
|
|
"num_tokens": 70348480.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1189,
|
|
"step_time": 50.34474147018045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1072.0,
|
|
"completions/max_terminated_length": 1072.0,
|
|
"completions/mean_length": 726.984375,
|
|
"completions/mean_terminated_length": 726.984375,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.8889088835567236,
|
|
"epoch": 0.6746031746031746,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.01804388262098655,
|
|
"learning_rate": 3.2596371882086167e-07,
|
|
"loss": 0.0123,
|
|
"num_tokens": 70406167.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1190,
|
|
"step_time": 38.42306902632117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1298.0,
|
|
"completions/max_terminated_length": 1298.0,
|
|
"completions/mean_length": 712.140625,
|
|
"completions/mean_terminated_length": 712.140625,
|
|
"completions/min_length": 288.0,
|
|
"completions/min_terminated_length": 288.0,
|
|
"entropy": 0.9873801618814468,
|
|
"epoch": 0.6751700680272109,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.010546976380283013,
|
|
"learning_rate": 3.2539682539682537e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 70463272.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1191,
|
|
"step_time": 46.42024501506239
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1573.0,
|
|
"completions/max_terminated_length": 1573.0,
|
|
"completions/mean_length": 757.015625,
|
|
"completions/mean_terminated_length": 757.015625,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.7312531173229218,
|
|
"epoch": 0.6757369614512472,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.009848068439168856,
|
|
"learning_rate": 3.248299319727891e-07,
|
|
"loss": 0.0171,
|
|
"num_tokens": 70524401.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1192,
|
|
"step_time": 58.316812553443015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1619.0,
|
|
"completions/max_terminated_length": 1619.0,
|
|
"completions/mean_length": 792.265625,
|
|
"completions/mean_terminated_length": 792.265625,
|
|
"completions/min_length": 292.0,
|
|
"completions/min_terminated_length": 292.0,
|
|
"entropy": 0.7126465123146772,
|
|
"epoch": 0.6763038548752834,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.43359375,
|
|
"kl": 0.0071313984371954575,
|
|
"learning_rate": 3.242630385487528e-07,
|
|
"loss": -0.0162,
|
|
"num_tokens": 70585338.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1193,
|
|
"step_time": 56.995766477659345
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1263.0,
|
|
"completions/max_terminated_length": 1263.0,
|
|
"completions/mean_length": 763.125,
|
|
"completions/mean_terminated_length": 763.125,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.7699721604585648,
|
|
"epoch": 0.6768707482993197,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.0066073409107048064,
|
|
"learning_rate": 3.2369614512471656e-07,
|
|
"loss": -0.005,
|
|
"num_tokens": 70642274.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.907933235168457,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1194,
|
|
"step_time": 44.40234444849193
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1789.0,
|
|
"completions/max_terminated_length": 1789.0,
|
|
"completions/mean_length": 614.34375,
|
|
"completions/mean_terminated_length": 614.34375,
|
|
"completions/min_length": 32.0,
|
|
"completions/min_terminated_length": 32.0,
|
|
"entropy": 0.794089587405324,
|
|
"epoch": 0.677437641723356,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.66015625,
|
|
"kl": 0.011897146963747218,
|
|
"learning_rate": 3.2312925170068026e-07,
|
|
"loss": 0.0011,
|
|
"num_tokens": 70691680.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7739239931106567,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1195,
|
|
"step_time": 63.48209861386567
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1609.0,
|
|
"completions/max_terminated_length": 1609.0,
|
|
"completions/mean_length": 723.671875,
|
|
"completions/mean_terminated_length": 723.671875,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 0.8111245725303888,
|
|
"epoch": 0.6780045351473923,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.439453125,
|
|
"kl": 0.012697158905211836,
|
|
"learning_rate": 3.2256235827664395e-07,
|
|
"loss": 0.0097,
|
|
"num_tokens": 70747203.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1196,
|
|
"step_time": 56.7683574417606
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1182.0,
|
|
"completions/max_terminated_length": 1182.0,
|
|
"completions/mean_length": 792.8125,
|
|
"completions/mean_terminated_length": 792.8125,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.786299604922533,
|
|
"epoch": 0.6785714285714286,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.275390625,
|
|
"kl": 0.006842593749752268,
|
|
"learning_rate": 3.219954648526077e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 70808703.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1197,
|
|
"step_time": 42.3951351782307
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1393.0,
|
|
"completions/max_terminated_length": 1393.0,
|
|
"completions/mean_length": 893.921875,
|
|
"completions/mean_terminated_length": 893.921875,
|
|
"completions/min_length": 543.0,
|
|
"completions/min_terminated_length": 543.0,
|
|
"entropy": 0.6661716923117638,
|
|
"epoch": 0.6791383219954649,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.005924049546592869,
|
|
"learning_rate": 3.2142857142857145e-07,
|
|
"loss": -0.0197,
|
|
"num_tokens": 70876546.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1198,
|
|
"step_time": 49.841370089910924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1363.0,
|
|
"completions/max_terminated_length": 1363.0,
|
|
"completions/mean_length": 726.359375,
|
|
"completions/mean_terminated_length": 726.359375,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.8698873035609722,
|
|
"epoch": 0.6797052154195011,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.65625,
|
|
"kl": 0.010281710478011519,
|
|
"learning_rate": 3.208616780045351e-07,
|
|
"loss": -0.0254,
|
|
"num_tokens": 70932689.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6871842741966248,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1199,
|
|
"step_time": 48.122578509151936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2029.0,
|
|
"completions/max_terminated_length": 2029.0,
|
|
"completions/mean_length": 829.75,
|
|
"completions/mean_terminated_length": 829.75,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.678981076925993,
|
|
"epoch": 0.6802721088435374,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.006873315098346211,
|
|
"learning_rate": 3.2029478458049884e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 70995425.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1200,
|
|
"step_time": 72.28340507950634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1628.0,
|
|
"completions/max_terminated_length": 1628.0,
|
|
"completions/mean_length": 775.9375,
|
|
"completions/mean_terminated_length": 775.9375,
|
|
"completions/min_length": 322.0,
|
|
"completions/min_terminated_length": 322.0,
|
|
"entropy": 0.7851563096046448,
|
|
"epoch": 0.6808390022675737,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.007131770529667847,
|
|
"learning_rate": 3.197278911564626e-07,
|
|
"loss": 0.0639,
|
|
"num_tokens": 71054037.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1201,
|
|
"step_time": 57.967341443523765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2711.0,
|
|
"completions/max_terminated_length": 2711.0,
|
|
"completions/mean_length": 780.03125,
|
|
"completions/mean_terminated_length": 780.03125,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.7905701491981745,
|
|
"epoch": 0.68140589569161,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.005879878590349108,
|
|
"learning_rate": 3.191609977324263e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 71114551.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 1202,
|
|
"step_time": 103.9807007741183
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1857.0,
|
|
"completions/max_terminated_length": 1857.0,
|
|
"completions/mean_length": 774.359375,
|
|
"completions/mean_terminated_length": 774.359375,
|
|
"completions/min_length": 457.0,
|
|
"completions/min_terminated_length": 457.0,
|
|
"entropy": 0.7334150485694408,
|
|
"epoch": 0.6819727891156463,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.006408129193005152,
|
|
"learning_rate": 3.1859410430839e-07,
|
|
"loss": 0.0078,
|
|
"num_tokens": 71173198.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.774084210395813,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1203,
|
|
"step_time": 65.37064866069704
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1414.0,
|
|
"completions/max_terminated_length": 1414.0,
|
|
"completions/mean_length": 725.375,
|
|
"completions/mean_terminated_length": 725.375,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.9488033540546894,
|
|
"epoch": 0.6825396825396826,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.322265625,
|
|
"kl": 0.007840074089472182,
|
|
"learning_rate": 3.1802721088435374e-07,
|
|
"loss": 0.0074,
|
|
"num_tokens": 71230782.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1204,
|
|
"step_time": 50.18996137473732
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2248.0,
|
|
"completions/max_terminated_length": 2248.0,
|
|
"completions/mean_length": 798.78125,
|
|
"completions/mean_terminated_length": 798.78125,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.7444244492799044,
|
|
"epoch": 0.6831065759637188,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.013879503385396674,
|
|
"learning_rate": 3.1746031746031743e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 71292960.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1205,
|
|
"step_time": 81.99354017339647
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1166.0,
|
|
"completions/max_terminated_length": 1166.0,
|
|
"completions/mean_length": 765.640625,
|
|
"completions/mean_terminated_length": 765.640625,
|
|
"completions/min_length": 446.0,
|
|
"completions/min_terminated_length": 446.0,
|
|
"entropy": 0.834197785705328,
|
|
"epoch": 0.6836734693877551,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.578125,
|
|
"kl": 0.007862241676775739,
|
|
"learning_rate": 3.168934240362812e-07,
|
|
"loss": 0.0164,
|
|
"num_tokens": 71351321.0,
|
|
"reward": 0.5234375,
|
|
"reward_std": 0.8927827477455139,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1206,
|
|
"step_time": 41.15044671762735
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1180.0,
|
|
"completions/max_terminated_length": 1180.0,
|
|
"completions/mean_length": 769.59375,
|
|
"completions/mean_terminated_length": 769.59375,
|
|
"completions/min_length": 304.0,
|
|
"completions/min_terminated_length": 304.0,
|
|
"entropy": 0.7829247303307056,
|
|
"epoch": 0.6842403628117913,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.015310686358134262,
|
|
"learning_rate": 3.163265306122449e-07,
|
|
"loss": 0.0209,
|
|
"num_tokens": 71411599.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1207,
|
|
"step_time": 41.876153447665274
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1351.0,
|
|
"completions/max_terminated_length": 1351.0,
|
|
"completions/mean_length": 710.5,
|
|
"completions/mean_terminated_length": 710.5,
|
|
"completions/min_length": 178.0,
|
|
"completions/min_terminated_length": 178.0,
|
|
"entropy": 0.8284341841936111,
|
|
"epoch": 0.6848072562358276,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6484375,
|
|
"kl": 0.009428308578208089,
|
|
"learning_rate": 3.1575963718820857e-07,
|
|
"loss": 0.0326,
|
|
"num_tokens": 71467807.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1208,
|
|
"step_time": 47.87776697054505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2591.0,
|
|
"completions/max_terminated_length": 2591.0,
|
|
"completions/mean_length": 911.953125,
|
|
"completions/mean_terminated_length": 911.953125,
|
|
"completions/min_length": 264.0,
|
|
"completions/min_terminated_length": 264.0,
|
|
"entropy": 0.591384882107377,
|
|
"epoch": 0.685374149659864,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.490234375,
|
|
"kl": 0.009712976607261226,
|
|
"learning_rate": 3.151927437641723e-07,
|
|
"loss": -0.037,
|
|
"num_tokens": 71536396.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1209,
|
|
"step_time": 95.9487595455721
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1328.0,
|
|
"completions/max_terminated_length": 1328.0,
|
|
"completions/mean_length": 691.34375,
|
|
"completions/mean_terminated_length": 691.34375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.7025200538337231,
|
|
"epoch": 0.6859410430839002,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66796875,
|
|
"kl": 0.009190167475026101,
|
|
"learning_rate": 3.1462585034013607e-07,
|
|
"loss": -0.007,
|
|
"num_tokens": 71590826.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1210,
|
|
"step_time": 46.92870340496302
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1225.0,
|
|
"completions/max_terminated_length": 1225.0,
|
|
"completions/mean_length": 724.265625,
|
|
"completions/mean_terminated_length": 724.265625,
|
|
"completions/min_length": 19.0,
|
|
"completions/min_terminated_length": 19.0,
|
|
"entropy": 0.8007152155041695,
|
|
"epoch": 0.6865079365079365,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.023239395581185818,
|
|
"learning_rate": 3.140589569160997e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 71649387.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1211,
|
|
"step_time": 44.30155893974006
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1485.0,
|
|
"completions/max_terminated_length": 1485.0,
|
|
"completions/mean_length": 834.890625,
|
|
"completions/mean_terminated_length": 834.890625,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.7228634133934975,
|
|
"epoch": 0.6870748299319728,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.00611676208791323,
|
|
"learning_rate": 3.1349206349206346e-07,
|
|
"loss": -0.0006,
|
|
"num_tokens": 71712052.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.7466442584991455,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1212,
|
|
"step_time": 51.798994448035955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1702.0,
|
|
"completions/max_terminated_length": 1702.0,
|
|
"completions/mean_length": 704.09375,
|
|
"completions/mean_terminated_length": 704.09375,
|
|
"completions/min_length": 241.0,
|
|
"completions/min_terminated_length": 241.0,
|
|
"entropy": 0.7955361399799585,
|
|
"epoch": 0.687641723356009,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"kl": 0.0074287534662289545,
|
|
"learning_rate": 3.129251700680272e-07,
|
|
"loss": 0.0052,
|
|
"num_tokens": 71767138.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1213,
|
|
"step_time": 60.05704145785421
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1328.0,
|
|
"completions/max_terminated_length": 1328.0,
|
|
"completions/mean_length": 738.703125,
|
|
"completions/mean_terminated_length": 738.703125,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 0.9776529334485531,
|
|
"epoch": 0.6882086167800453,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.31640625,
|
|
"kl": 0.012538022740045562,
|
|
"learning_rate": 3.123582766439909e-07,
|
|
"loss": 0.0047,
|
|
"num_tokens": 71824527.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1214,
|
|
"step_time": 46.786511175334454
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1092.0,
|
|
"completions/max_terminated_length": 1092.0,
|
|
"completions/mean_length": 630.46875,
|
|
"completions/mean_terminated_length": 630.46875,
|
|
"completions/min_length": 129.0,
|
|
"completions/min_terminated_length": 129.0,
|
|
"entropy": 1.013225018978119,
|
|
"epoch": 0.6887755102040817,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.439453125,
|
|
"kl": 0.015821554436115548,
|
|
"learning_rate": 3.117913832199546e-07,
|
|
"loss": -0.0018,
|
|
"num_tokens": 71878085.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5107961297035217,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1215,
|
|
"step_time": 40.298684887588024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1692.0,
|
|
"completions/max_terminated_length": 1692.0,
|
|
"completions/mean_length": 767.3125,
|
|
"completions/mean_terminated_length": 767.3125,
|
|
"completions/min_length": 230.0,
|
|
"completions/min_terminated_length": 230.0,
|
|
"entropy": 0.6991136409342289,
|
|
"epoch": 0.6893424036281179,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.007220871717436239,
|
|
"learning_rate": 3.1122448979591836e-07,
|
|
"loss": 0.0215,
|
|
"num_tokens": 71937673.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1216,
|
|
"step_time": 59.50843483582139
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1259.0,
|
|
"completions/max_terminated_length": 1259.0,
|
|
"completions/mean_length": 755.0625,
|
|
"completions/mean_terminated_length": 755.0625,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.9117769785225391,
|
|
"epoch": 0.6899092970521542,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00537109375,
|
|
"kl": 0.006100863727624528,
|
|
"learning_rate": 3.1065759637188205e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 71996797.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.1666666716337204,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1217,
|
|
"step_time": 44.962114319205284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1338.0,
|
|
"completions/max_terminated_length": 1338.0,
|
|
"completions/mean_length": 848.890625,
|
|
"completions/mean_terminated_length": 848.890625,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.7357565108686686,
|
|
"epoch": 0.6904761904761905,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.009637762530473992,
|
|
"learning_rate": 3.100907029478458e-07,
|
|
"loss": -0.0037,
|
|
"num_tokens": 72060270.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1218,
|
|
"step_time": 47.018812905065715
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1420.0,
|
|
"completions/max_terminated_length": 1420.0,
|
|
"completions/mean_length": 857.234375,
|
|
"completions/mean_terminated_length": 857.234375,
|
|
"completions/min_length": 473.0,
|
|
"completions/min_terminated_length": 473.0,
|
|
"entropy": 0.7382922917604446,
|
|
"epoch": 0.6910430839002267,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.00478549039689824,
|
|
"learning_rate": 3.095238095238095e-07,
|
|
"loss": -0.0085,
|
|
"num_tokens": 72124661.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6542273163795471,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1219,
|
|
"step_time": 50.681492526084185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1592.0,
|
|
"completions/max_terminated_length": 1592.0,
|
|
"completions/mean_length": 756.671875,
|
|
"completions/mean_terminated_length": 756.671875,
|
|
"completions/min_length": 248.0,
|
|
"completions/min_terminated_length": 248.0,
|
|
"entropy": 0.6995925661176443,
|
|
"epoch": 0.691609977324263,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5859375,
|
|
"kl": 0.012112914206227288,
|
|
"learning_rate": 3.089569160997732e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 72185240.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.8164966106414795,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1220,
|
|
"step_time": 57.76277104858309
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1151.0,
|
|
"completions/max_terminated_length": 1151.0,
|
|
"completions/mean_length": 741.34375,
|
|
"completions/mean_terminated_length": 741.34375,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.8239504024386406,
|
|
"epoch": 0.6921768707482994,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.458984375,
|
|
"kl": 0.005739541811635718,
|
|
"learning_rate": 3.0839002267573694e-07,
|
|
"loss": 0.0058,
|
|
"num_tokens": 72242374.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1221,
|
|
"step_time": 41.083685249090195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1632.0,
|
|
"completions/max_terminated_length": 1632.0,
|
|
"completions/mean_length": 752.1875,
|
|
"completions/mean_terminated_length": 752.1875,
|
|
"completions/min_length": 171.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"entropy": 0.7050927616655827,
|
|
"epoch": 0.6927437641723356,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"kl": 0.012849961305619217,
|
|
"learning_rate": 3.078231292517007e-07,
|
|
"loss": 0.0147,
|
|
"num_tokens": 72301914.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1222,
|
|
"step_time": 58.26110740378499
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1051.0,
|
|
"completions/max_terminated_length": 1051.0,
|
|
"completions/mean_length": 720.671875,
|
|
"completions/mean_terminated_length": 720.671875,
|
|
"completions/min_length": 428.0,
|
|
"completions/min_terminated_length": 428.0,
|
|
"entropy": 0.9347487296909094,
|
|
"epoch": 0.6933106575963719,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.010723743442213163,
|
|
"learning_rate": 3.072562358276644e-07,
|
|
"loss": 0.0152,
|
|
"num_tokens": 72358853.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1223,
|
|
"step_time": 37.423720388673246
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1477.0,
|
|
"completions/max_terminated_length": 1477.0,
|
|
"completions/mean_length": 662.703125,
|
|
"completions/mean_terminated_length": 662.703125,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 0.7324567809700966,
|
|
"epoch": 0.6938775510204082,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6796875,
|
|
"kl": 0.010438743673148565,
|
|
"learning_rate": 3.066893424036281e-07,
|
|
"loss": -0.0109,
|
|
"num_tokens": 72412026.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.7983350157737732,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1224,
|
|
"step_time": 52.19476579595357
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1151.0,
|
|
"completions/max_terminated_length": 1151.0,
|
|
"completions/mean_length": 611.953125,
|
|
"completions/mean_terminated_length": 611.953125,
|
|
"completions/min_length": 207.0,
|
|
"completions/min_terminated_length": 207.0,
|
|
"entropy": 0.8020634986460209,
|
|
"epoch": 0.6944444444444444,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.027214992471272126,
|
|
"learning_rate": 3.0612244897959183e-07,
|
|
"loss": 0.0114,
|
|
"num_tokens": 72461039.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1225,
|
|
"step_time": 40.33260082732886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1417.0,
|
|
"completions/mean_length": 766.609375,
|
|
"completions/mean_terminated_length": 713.761962890625,
|
|
"completions/min_length": 181.0,
|
|
"completions/min_terminated_length": 181.0,
|
|
"entropy": 0.8936783596873283,
|
|
"epoch": 0.6950113378684807,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4140625,
|
|
"kl": 0.008530761464498937,
|
|
"learning_rate": 3.055555555555556e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 72522598.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1226,
|
|
"step_time": 180.23161055147648
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 724.484375,
|
|
"completions/mean_terminated_length": 724.484375,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.8287078030407429,
|
|
"epoch": 0.6955782312925171,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.013218163876445033,
|
|
"learning_rate": 3.049886621315192e-07,
|
|
"loss": 0.0349,
|
|
"num_tokens": 72582245.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1227,
|
|
"step_time": 49.550456166267395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1466.0,
|
|
"completions/max_terminated_length": 1466.0,
|
|
"completions/mean_length": 770.296875,
|
|
"completions/mean_terminated_length": 770.296875,
|
|
"completions/min_length": 416.0,
|
|
"completions/min_terminated_length": 416.0,
|
|
"entropy": 0.8143008500337601,
|
|
"epoch": 0.6961451247165533,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"kl": 0.006532995234010741,
|
|
"learning_rate": 3.04421768707483e-07,
|
|
"loss": 0.019,
|
|
"num_tokens": 72643712.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1228,
|
|
"step_time": 52.44228432234377
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1712.0,
|
|
"completions/max_terminated_length": 1712.0,
|
|
"completions/mean_length": 802.1875,
|
|
"completions/mean_terminated_length": 802.1875,
|
|
"completions/min_length": 238.0,
|
|
"completions/min_terminated_length": 238.0,
|
|
"entropy": 0.7020249702036381,
|
|
"epoch": 0.6967120181405896,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.005149896314833313,
|
|
"learning_rate": 3.038548752834467e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 72706556.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1229,
|
|
"step_time": 61.80100727267563
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1133.0,
|
|
"completions/max_terminated_length": 1133.0,
|
|
"completions/mean_length": 689.0625,
|
|
"completions/mean_terminated_length": 689.0625,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.7375645842403173,
|
|
"epoch": 0.6972789115646258,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.00913719623349607,
|
|
"learning_rate": 3.032879818594104e-07,
|
|
"loss": -0.0249,
|
|
"num_tokens": 72761064.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1230,
|
|
"step_time": 39.970598075538874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1371.0,
|
|
"completions/max_terminated_length": 1371.0,
|
|
"completions/mean_length": 589.9375,
|
|
"completions/mean_terminated_length": 589.9375,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 0.8049846533685923,
|
|
"epoch": 0.6978458049886621,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6953125,
|
|
"kl": 0.011146385280881077,
|
|
"learning_rate": 3.027210884353741e-07,
|
|
"loss": 0.0264,
|
|
"num_tokens": 72813300.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.8164966106414795,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1231,
|
|
"step_time": 52.04534669499844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1327.0,
|
|
"completions/max_terminated_length": 1327.0,
|
|
"completions/mean_length": 668.203125,
|
|
"completions/mean_terminated_length": 668.203125,
|
|
"completions/min_length": 29.0,
|
|
"completions/min_terminated_length": 29.0,
|
|
"entropy": 0.9286523349583149,
|
|
"epoch": 0.6984126984126984,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.007829043694073334,
|
|
"learning_rate": 3.0215419501133787e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 72867049.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.666620135307312,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1232,
|
|
"step_time": 46.81675331201404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1721.0,
|
|
"completions/max_terminated_length": 1721.0,
|
|
"completions/mean_length": 798.546875,
|
|
"completions/mean_terminated_length": 798.546875,
|
|
"completions/min_length": 428.0,
|
|
"completions/min_terminated_length": 428.0,
|
|
"entropy": 0.9234435819089413,
|
|
"epoch": 0.6989795918367347,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.010452065122080967,
|
|
"learning_rate": 3.0158730158730156e-07,
|
|
"loss": -0.0023,
|
|
"num_tokens": 72929476.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1233,
|
|
"step_time": 61.33797580841929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1435.0,
|
|
"completions/max_terminated_length": 1435.0,
|
|
"completions/mean_length": 774.015625,
|
|
"completions/mean_terminated_length": 774.015625,
|
|
"completions/min_length": 114.0,
|
|
"completions/min_terminated_length": 114.0,
|
|
"entropy": 0.7512411493808031,
|
|
"epoch": 0.699546485260771,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.02342989417957142,
|
|
"learning_rate": 3.010204081632653e-07,
|
|
"loss": -0.0334,
|
|
"num_tokens": 72990861.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1234,
|
|
"step_time": 51.37074568029493
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1565.0,
|
|
"completions/max_terminated_length": 1565.0,
|
|
"completions/mean_length": 683.34375,
|
|
"completions/mean_terminated_length": 683.34375,
|
|
"completions/min_length": 112.0,
|
|
"completions/min_terminated_length": 112.0,
|
|
"entropy": 0.7321684081107378,
|
|
"epoch": 0.7001133786848073,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.00959622286609374,
|
|
"learning_rate": 3.00453514739229e-07,
|
|
"loss": -0.0023,
|
|
"num_tokens": 73047675.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6423677802085876,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1235,
|
|
"step_time": 57.0398217337206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1367.0,
|
|
"completions/max_terminated_length": 1367.0,
|
|
"completions/mean_length": 749.609375,
|
|
"completions/mean_terminated_length": 749.609375,
|
|
"completions/min_length": 350.0,
|
|
"completions/min_terminated_length": 350.0,
|
|
"entropy": 1.0315840356051922,
|
|
"epoch": 0.7006802721088435,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6015625,
|
|
"kl": 0.007061148717184551,
|
|
"learning_rate": 2.998866213151927e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 73107050.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1236,
|
|
"step_time": 48.65876642614603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1460.0,
|
|
"completions/max_terminated_length": 1460.0,
|
|
"completions/mean_length": 783.078125,
|
|
"completions/mean_terminated_length": 783.078125,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.7931034341454506,
|
|
"epoch": 0.7012471655328798,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.65234375,
|
|
"kl": 0.007388332771370187,
|
|
"learning_rate": 2.9931972789115645e-07,
|
|
"loss": 0.0525,
|
|
"num_tokens": 73166607.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7516103982925415,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1237,
|
|
"step_time": 51.18390062917024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1609.0,
|
|
"completions/max_terminated_length": 1609.0,
|
|
"completions/mean_length": 708.3125,
|
|
"completions/mean_terminated_length": 708.3125,
|
|
"completions/min_length": 338.0,
|
|
"completions/min_terminated_length": 338.0,
|
|
"entropy": 1.1367200873792171,
|
|
"epoch": 0.7018140589569161,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.310546875,
|
|
"kl": 0.007605911057908088,
|
|
"learning_rate": 2.987528344671202e-07,
|
|
"loss": 0.0089,
|
|
"num_tokens": 73221531.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1238,
|
|
"step_time": 56.485504480078816
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1652.0,
|
|
"completions/max_terminated_length": 1652.0,
|
|
"completions/mean_length": 697.90625,
|
|
"completions/mean_terminated_length": 697.90625,
|
|
"completions/min_length": 203.0,
|
|
"completions/min_terminated_length": 203.0,
|
|
"entropy": 0.8720447905361652,
|
|
"epoch": 0.7023809523809523,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.01731234375620261,
|
|
"learning_rate": 2.9818594104308385e-07,
|
|
"loss": 0.0047,
|
|
"num_tokens": 73276829.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3435921370983124,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1239,
|
|
"step_time": 58.400853395462036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1276.0,
|
|
"completions/max_terminated_length": 1276.0,
|
|
"completions/mean_length": 729.34375,
|
|
"completions/mean_terminated_length": 729.34375,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"entropy": 0.6715658456087112,
|
|
"epoch": 0.7029478458049887,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.412109375,
|
|
"kl": 0.014108914881944656,
|
|
"learning_rate": 2.976190476190476e-07,
|
|
"loss": 0.0349,
|
|
"num_tokens": 73333499.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1240,
|
|
"step_time": 44.95885122194886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1546.0,
|
|
"completions/max_terminated_length": 1546.0,
|
|
"completions/mean_length": 773.421875,
|
|
"completions/mean_terminated_length": 773.421875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.7311521042138338,
|
|
"epoch": 0.703514739229025,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.00813031340658199,
|
|
"learning_rate": 2.9705215419501134e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 73393702.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1241,
|
|
"step_time": 54.81931927148253
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2057.0,
|
|
"completions/max_terminated_length": 2057.0,
|
|
"completions/mean_length": 807.546875,
|
|
"completions/mean_terminated_length": 807.546875,
|
|
"completions/min_length": 422.0,
|
|
"completions/min_terminated_length": 422.0,
|
|
"entropy": 0.8853982463479042,
|
|
"epoch": 0.7040816326530612,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.390625,
|
|
"kl": 0.016378501561121084,
|
|
"learning_rate": 2.9648526077097504e-07,
|
|
"loss": 0.0123,
|
|
"num_tokens": 73454865.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1242,
|
|
"step_time": 73.07493476383388
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1150.0,
|
|
"completions/max_terminated_length": 1150.0,
|
|
"completions/mean_length": 761.609375,
|
|
"completions/mean_terminated_length": 761.609375,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 1.0804704800248146,
|
|
"epoch": 0.7046485260770975,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.009561734317685477,
|
|
"learning_rate": 2.9591836734693874e-07,
|
|
"loss": -0.0062,
|
|
"num_tokens": 73514192.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1243,
|
|
"step_time": 40.904444872401655
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1633.0,
|
|
"completions/max_terminated_length": 1633.0,
|
|
"completions/mean_length": 884.03125,
|
|
"completions/mean_terminated_length": 884.03125,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.6861023455858231,
|
|
"epoch": 0.7052154195011338,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.019731883934582584,
|
|
"learning_rate": 2.953514739229025e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 73582466.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1244,
|
|
"step_time": 57.441369350999594
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1425.0,
|
|
"completions/mean_length": 670.71875,
|
|
"completions/mean_terminated_length": 616.3492431640625,
|
|
"completions/min_length": 96.0,
|
|
"completions/min_terminated_length": 96.0,
|
|
"entropy": 0.8277360424399376,
|
|
"epoch": 0.70578231292517,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.67578125,
|
|
"kl": 0.00828776771959383,
|
|
"learning_rate": 2.947845804988662e-07,
|
|
"loss": -0.0537,
|
|
"num_tokens": 73633592.0,
|
|
"reward": 0.65625,
|
|
"reward_std": 0.9464846849441528,
|
|
"rewards/accuracy_reward_func/mean": 0.59375,
|
|
"rewards/accuracy_reward_func/std": 0.9209855198860168,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1245,
|
|
"step_time": 169.73814003542066
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1535.0,
|
|
"completions/max_terminated_length": 1535.0,
|
|
"completions/mean_length": 788.84375,
|
|
"completions/mean_terminated_length": 788.84375,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.748440695926547,
|
|
"epoch": 0.7063492063492064,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.244140625,
|
|
"kl": 0.006073269003536552,
|
|
"learning_rate": 2.9421768707482993e-07,
|
|
"loss": -0.0062,
|
|
"num_tokens": 73692870.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1246,
|
|
"step_time": 53.85055844578892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1321.0,
|
|
"completions/max_terminated_length": 1321.0,
|
|
"completions/mean_length": 823.046875,
|
|
"completions/mean_terminated_length": 823.046875,
|
|
"completions/min_length": 528.0,
|
|
"completions/min_terminated_length": 528.0,
|
|
"entropy": 0.6589046604931355,
|
|
"epoch": 0.7069160997732427,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2109375,
|
|
"kl": 0.017391609668266028,
|
|
"learning_rate": 2.9365079365079363e-07,
|
|
"loss": -0.0065,
|
|
"num_tokens": 73757697.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1247,
|
|
"step_time": 48.11004972830415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1365.0,
|
|
"completions/mean_length": 880.0625,
|
|
"completions/mean_terminated_length": 829.0159301757812,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"entropy": 0.6327879559248686,
|
|
"epoch": 0.7074829931972789,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.48046875,
|
|
"kl": 0.0059249412443023175,
|
|
"learning_rate": 2.930839002267573e-07,
|
|
"loss": -0.0181,
|
|
"num_tokens": 73823877.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7531349658966064,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1248,
|
|
"step_time": 170.95728433597833
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3571.0,
|
|
"completions/max_terminated_length": 3571.0,
|
|
"completions/mean_length": 767.421875,
|
|
"completions/mean_terminated_length": 767.421875,
|
|
"completions/min_length": 370.0,
|
|
"completions/min_terminated_length": 370.0,
|
|
"entropy": 0.7562596760690212,
|
|
"epoch": 0.7080498866213152,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.01149018295109272,
|
|
"learning_rate": 2.925170068027211e-07,
|
|
"loss": -0.0283,
|
|
"num_tokens": 73883488.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7007648944854736,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1249,
|
|
"step_time": 145.13862953800708
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1234.0,
|
|
"completions/mean_length": 758.3125,
|
|
"completions/mean_terminated_length": 705.3333740234375,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.8107492886483669,
|
|
"epoch": 0.7086167800453514,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40625,
|
|
"kl": 0.01330675242934376,
|
|
"learning_rate": 2.919501133786848e-07,
|
|
"loss": 0.0271,
|
|
"num_tokens": 73942388.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1250,
|
|
"step_time": 173.05524683278054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1181.0,
|
|
"completions/max_terminated_length": 1181.0,
|
|
"completions/mean_length": 721.109375,
|
|
"completions/mean_terminated_length": 721.109375,
|
|
"completions/min_length": 409.0,
|
|
"completions/min_terminated_length": 409.0,
|
|
"entropy": 0.9488244727253914,
|
|
"epoch": 0.7091836734693877,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.006170788619783707,
|
|
"learning_rate": 2.913832199546485e-07,
|
|
"loss": 0.0049,
|
|
"num_tokens": 74000059.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1251,
|
|
"step_time": 41.95774591527879
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1287.0,
|
|
"completions/max_terminated_length": 1287.0,
|
|
"completions/mean_length": 710.09375,
|
|
"completions/mean_terminated_length": 710.09375,
|
|
"completions/min_length": 208.0,
|
|
"completions/min_terminated_length": 208.0,
|
|
"entropy": 0.719386400654912,
|
|
"epoch": 0.7097505668934241,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34375,
|
|
"kl": 0.006540272006532177,
|
|
"learning_rate": 2.908163265306122e-07,
|
|
"loss": -0.0034,
|
|
"num_tokens": 74055841.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1252,
|
|
"step_time": 45.79248642269522
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1133.0,
|
|
"completions/mean_length": 786.75,
|
|
"completions/mean_terminated_length": 734.2222900390625,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.8479833174496889,
|
|
"epoch": 0.7103174603174603,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.03237981146958191,
|
|
"learning_rate": 2.9024943310657596e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 74116417.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1253,
|
|
"step_time": 171.28832686785609
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1495.0,
|
|
"completions/max_terminated_length": 1495.0,
|
|
"completions/mean_length": 702.0,
|
|
"completions/mean_terminated_length": 702.0,
|
|
"completions/min_length": 332.0,
|
|
"completions/min_terminated_length": 332.0,
|
|
"entropy": 0.9278449863195419,
|
|
"epoch": 0.7108843537414966,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.009528935421258211,
|
|
"learning_rate": 2.896825396825397e-07,
|
|
"loss": 0.0147,
|
|
"num_tokens": 74173065.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1254,
|
|
"step_time": 53.834491525776684
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1450.0,
|
|
"completions/max_terminated_length": 1450.0,
|
|
"completions/mean_length": 630.59375,
|
|
"completions/mean_terminated_length": 630.59375,
|
|
"completions/min_length": 181.0,
|
|
"completions/min_terminated_length": 181.0,
|
|
"entropy": 0.8112505786120892,
|
|
"epoch": 0.7114512471655329,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.671875,
|
|
"kl": 0.009214796664309688,
|
|
"learning_rate": 2.8911564625850336e-07,
|
|
"loss": 0.0291,
|
|
"num_tokens": 74229079.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1255,
|
|
"step_time": 56.24102068506181
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1432.0,
|
|
"completions/max_terminated_length": 1432.0,
|
|
"completions/mean_length": 657.65625,
|
|
"completions/mean_terminated_length": 657.65625,
|
|
"completions/min_length": 271.0,
|
|
"completions/min_terminated_length": 271.0,
|
|
"entropy": 0.7663573008030653,
|
|
"epoch": 0.7120181405895691,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"kl": 0.020211714261677116,
|
|
"learning_rate": 2.885487528344671e-07,
|
|
"loss": 0.0158,
|
|
"num_tokens": 74281369.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7927232980728149,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1256,
|
|
"step_time": 50.5737309390679
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1291.0,
|
|
"completions/max_terminated_length": 1291.0,
|
|
"completions/mean_length": 737.21875,
|
|
"completions/mean_terminated_length": 737.21875,
|
|
"completions/min_length": 392.0,
|
|
"completions/min_terminated_length": 392.0,
|
|
"entropy": 1.006922371685505,
|
|
"epoch": 0.7125850340136054,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.010737384232925251,
|
|
"learning_rate": 2.8798185941043086e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 74339063.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1257,
|
|
"step_time": 45.48856576345861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1628.0,
|
|
"completions/max_terminated_length": 1628.0,
|
|
"completions/mean_length": 729.890625,
|
|
"completions/mean_terminated_length": 729.890625,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.8311297688633204,
|
|
"epoch": 0.7131519274376418,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.010450679998029955,
|
|
"learning_rate": 2.8741496598639455e-07,
|
|
"loss": -0.0066,
|
|
"num_tokens": 74395184.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1258,
|
|
"step_time": 56.98518708720803
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1977.0,
|
|
"completions/max_terminated_length": 1977.0,
|
|
"completions/mean_length": 855.609375,
|
|
"completions/mean_terminated_length": 855.609375,
|
|
"completions/min_length": 411.0,
|
|
"completions/min_terminated_length": 411.0,
|
|
"entropy": 0.8867407329380512,
|
|
"epoch": 0.713718820861678,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.42578125,
|
|
"kl": 0.006427495391108096,
|
|
"learning_rate": 2.8684807256235825e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 74461431.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.53079754114151,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1259,
|
|
"step_time": 69.60293906647712
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1258.0,
|
|
"completions/max_terminated_length": 1258.0,
|
|
"completions/mean_length": 677.71875,
|
|
"completions/mean_terminated_length": 677.71875,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.728508373722434,
|
|
"epoch": 0.7142857142857143,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.011970607578405179,
|
|
"learning_rate": 2.86281179138322e-07,
|
|
"loss": -0.0189,
|
|
"num_tokens": 74514517.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1260,
|
|
"step_time": 43.64888298418373
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1225.0,
|
|
"completions/max_terminated_length": 1225.0,
|
|
"completions/mean_length": 674.484375,
|
|
"completions/mean_terminated_length": 674.484375,
|
|
"completions/min_length": 392.0,
|
|
"completions/min_terminated_length": 392.0,
|
|
"entropy": 1.0795543305575848,
|
|
"epoch": 0.7148526077097506,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.68359375,
|
|
"kl": 0.00881368838599883,
|
|
"learning_rate": 2.857142857142857e-07,
|
|
"loss": 0.0263,
|
|
"num_tokens": 74567620.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1261,
|
|
"step_time": 42.99827969633043
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1568.0,
|
|
"completions/max_terminated_length": 1568.0,
|
|
"completions/mean_length": 833.546875,
|
|
"completions/mean_terminated_length": 833.546875,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.6604405902326107,
|
|
"epoch": 0.7154195011337868,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.466796875,
|
|
"kl": 0.015640550504031125,
|
|
"learning_rate": 2.8514739229024944e-07,
|
|
"loss": 0.0004,
|
|
"num_tokens": 74633199.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.7325600981712341,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1262,
|
|
"step_time": 55.78884386084974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1233.0,
|
|
"completions/max_terminated_length": 1233.0,
|
|
"completions/mean_length": 691.34375,
|
|
"completions/mean_terminated_length": 691.34375,
|
|
"completions/min_length": 316.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"entropy": 0.9017839059233665,
|
|
"epoch": 0.7159863945578231,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.008396626188186929,
|
|
"learning_rate": 2.8458049886621314e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 74687309.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1263,
|
|
"step_time": 43.72132656350732
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1213.0,
|
|
"completions/max_terminated_length": 1213.0,
|
|
"completions/mean_length": 754.890625,
|
|
"completions/mean_terminated_length": 754.890625,
|
|
"completions/min_length": 336.0,
|
|
"completions/min_terminated_length": 336.0,
|
|
"entropy": 0.7075295206159353,
|
|
"epoch": 0.7165532879818595,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.006945651664864272,
|
|
"learning_rate": 2.8401360544217684e-07,
|
|
"loss": -0.001,
|
|
"num_tokens": 74744782.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.8318066596984863,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1264,
|
|
"step_time": 42.22118824534118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1216.0,
|
|
"completions/max_terminated_length": 1216.0,
|
|
"completions/mean_length": 701.453125,
|
|
"completions/mean_terminated_length": 701.453125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.810277197510004,
|
|
"epoch": 0.7171201814058957,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.006896903782035224,
|
|
"learning_rate": 2.834467120181406e-07,
|
|
"loss": 0.0256,
|
|
"num_tokens": 74799131.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1265,
|
|
"step_time": 43.60357501450926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1395.0,
|
|
"completions/max_terminated_length": 1395.0,
|
|
"completions/mean_length": 772.75,
|
|
"completions/mean_terminated_length": 772.75,
|
|
"completions/min_length": 390.0,
|
|
"completions/min_terminated_length": 390.0,
|
|
"entropy": 0.6054725553840399,
|
|
"epoch": 0.717687074829932,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.007965900120325387,
|
|
"learning_rate": 2.8287981859410433e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 74858339.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1266,
|
|
"step_time": 48.839362574741244
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1503.0,
|
|
"completions/max_terminated_length": 1503.0,
|
|
"completions/mean_length": 805.84375,
|
|
"completions/mean_terminated_length": 805.84375,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.5716987736523151,
|
|
"epoch": 0.7182539682539683,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.0059544534888118505,
|
|
"learning_rate": 2.82312925170068e-07,
|
|
"loss": 0.0325,
|
|
"num_tokens": 74920281.0,
|
|
"reward": 0.625,
|
|
"reward_std": 0.8997353911399841,
|
|
"rewards/accuracy_reward_func/mean": 0.5625,
|
|
"rewards/accuracy_reward_func/std": 0.9063270092010498,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1267,
|
|
"step_time": 53.234523847699165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1197.0,
|
|
"completions/max_terminated_length": 1197.0,
|
|
"completions/mean_length": 766.890625,
|
|
"completions/mean_terminated_length": 766.890625,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 0.9586678482592106,
|
|
"epoch": 0.7188208616780045,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.010775131522677839,
|
|
"learning_rate": 2.8174603174603173e-07,
|
|
"loss": -0.007,
|
|
"num_tokens": 74983442.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1268,
|
|
"step_time": 44.22962439991534
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1508.0,
|
|
"completions/max_terminated_length": 1508.0,
|
|
"completions/mean_length": 802.203125,
|
|
"completions/mean_terminated_length": 802.203125,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.5859830342233181,
|
|
"epoch": 0.7193877551020408,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46875,
|
|
"kl": 0.0221819829166634,
|
|
"learning_rate": 2.811791383219955e-07,
|
|
"loss": 0.0256,
|
|
"num_tokens": 75045487.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7516103982925415,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1269,
|
|
"step_time": 53.15816253144294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1030.0,
|
|
"completions/max_terminated_length": 1030.0,
|
|
"completions/mean_length": 613.046875,
|
|
"completions/mean_terminated_length": 613.046875,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.7558637242764235,
|
|
"epoch": 0.719954648526077,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.8671875,
|
|
"kl": 0.021848756907274947,
|
|
"learning_rate": 2.8061224489795917e-07,
|
|
"loss": -0.0286,
|
|
"num_tokens": 75096722.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7663560509681702,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1270,
|
|
"step_time": 37.486066572368145
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1217.0,
|
|
"completions/max_terminated_length": 1217.0,
|
|
"completions/mean_length": 709.984375,
|
|
"completions/mean_terminated_length": 709.984375,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.9835778512060642,
|
|
"epoch": 0.7205215419501134,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40234375,
|
|
"kl": 0.009228013470419683,
|
|
"learning_rate": 2.8004535147392287e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 75150545.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1271,
|
|
"step_time": 42.55790672171861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1919.0,
|
|
"completions/max_terminated_length": 1919.0,
|
|
"completions/mean_length": 667.890625,
|
|
"completions/mean_terminated_length": 667.890625,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.7406773120164871,
|
|
"epoch": 0.7210884353741497,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.82421875,
|
|
"kl": 0.01158401662542019,
|
|
"learning_rate": 2.794784580498866e-07,
|
|
"loss": 0.0341,
|
|
"num_tokens": 75203362.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8380447030067444,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1272,
|
|
"step_time": 67.93210411816835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1013.0,
|
|
"completions/max_terminated_length": 1013.0,
|
|
"completions/mean_length": 596.84375,
|
|
"completions/mean_terminated_length": 596.84375,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.8827281687408686,
|
|
"epoch": 0.721655328798186,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.671875,
|
|
"kl": 0.012829241881263442,
|
|
"learning_rate": 2.789115646258503e-07,
|
|
"loss": 0.0223,
|
|
"num_tokens": 75251664.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.7943251132965088,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1273,
|
|
"step_time": 35.71300198696554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1529.0,
|
|
"completions/max_terminated_length": 1529.0,
|
|
"completions/mean_length": 901.203125,
|
|
"completions/mean_terminated_length": 901.203125,
|
|
"completions/min_length": 403.0,
|
|
"completions/min_terminated_length": 403.0,
|
|
"entropy": 0.621662201359868,
|
|
"epoch": 0.7222222222222222,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.008862611473887227,
|
|
"learning_rate": 2.7834467120181406e-07,
|
|
"loss": -0.0249,
|
|
"num_tokens": 75320789.0,
|
|
"reward": 0.4140625,
|
|
"reward_std": 0.7794721722602844,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1274,
|
|
"step_time": 54.03482427075505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1455.0,
|
|
"completions/mean_length": 814.015625,
|
|
"completions/mean_terminated_length": 761.920654296875,
|
|
"completions/min_length": 289.0,
|
|
"completions/min_terminated_length": 289.0,
|
|
"entropy": 0.649225065484643,
|
|
"epoch": 0.7227891156462585,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.011527249123901129,
|
|
"learning_rate": 2.7777777777777776e-07,
|
|
"loss": -0.0106,
|
|
"num_tokens": 75383558.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.8199823498725891,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1275,
|
|
"step_time": 171.96160723548383
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1849.0,
|
|
"completions/max_terminated_length": 1849.0,
|
|
"completions/mean_length": 772.71875,
|
|
"completions/mean_terminated_length": 772.71875,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.7997859735041857,
|
|
"epoch": 0.7233560090702947,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58203125,
|
|
"kl": 0.008677219608216546,
|
|
"learning_rate": 2.7721088435374146e-07,
|
|
"loss": 0.0115,
|
|
"num_tokens": 75442540.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1276,
|
|
"step_time": 64.61498490907252
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1237.0,
|
|
"completions/max_terminated_length": 1237.0,
|
|
"completions/mean_length": 713.171875,
|
|
"completions/mean_terminated_length": 713.171875,
|
|
"completions/min_length": 186.0,
|
|
"completions/min_terminated_length": 186.0,
|
|
"entropy": 0.7881220132112503,
|
|
"epoch": 0.7239229024943311,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.010341452158172615,
|
|
"learning_rate": 2.766439909297052e-07,
|
|
"loss": -0.0147,
|
|
"num_tokens": 75499399.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1277,
|
|
"step_time": 44.12419076543301
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1052.0,
|
|
"completions/max_terminated_length": 1052.0,
|
|
"completions/mean_length": 652.234375,
|
|
"completions/mean_terminated_length": 652.234375,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 1.0551521927118301,
|
|
"epoch": 0.7244897959183674,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.72265625,
|
|
"kl": 0.015491795784328133,
|
|
"learning_rate": 2.7607709750566895e-07,
|
|
"loss": 0.0211,
|
|
"num_tokens": 75552238.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1278,
|
|
"step_time": 37.41226695012301
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1377.0,
|
|
"completions/max_terminated_length": 1377.0,
|
|
"completions/mean_length": 689.078125,
|
|
"completions/mean_terminated_length": 689.078125,
|
|
"completions/min_length": 335.0,
|
|
"completions/min_terminated_length": 335.0,
|
|
"entropy": 0.7591619677841663,
|
|
"epoch": 0.7250566893424036,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"kl": 0.008736869174754247,
|
|
"learning_rate": 2.755102040816326e-07,
|
|
"loss": -0.0051,
|
|
"num_tokens": 75604699.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5033986568450928,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.1015625,
|
|
"rewards/format_reward_func/std": 0.20275264978408813,
|
|
"step": 1279,
|
|
"step_time": 48.25969914998859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1053.0,
|
|
"completions/max_terminated_length": 1053.0,
|
|
"completions/mean_length": 732.40625,
|
|
"completions/mean_terminated_length": 732.40625,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7826445735991001,
|
|
"epoch": 0.7256235827664399,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.0072278706065844744,
|
|
"learning_rate": 2.7494331065759635e-07,
|
|
"loss": 0.0097,
|
|
"num_tokens": 75662429.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1280,
|
|
"step_time": 37.36694760993123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1259.0,
|
|
"completions/max_terminated_length": 1259.0,
|
|
"completions/mean_length": 650.796875,
|
|
"completions/mean_terminated_length": 650.796875,
|
|
"completions/min_length": 333.0,
|
|
"completions/min_terminated_length": 333.0,
|
|
"entropy": 0.7009071428328753,
|
|
"epoch": 0.7261904761904762,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.625,
|
|
"kl": 0.009031634108396247,
|
|
"learning_rate": 2.743764172335601e-07,
|
|
"loss": -0.0083,
|
|
"num_tokens": 75717584.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7393422722816467,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1281,
|
|
"step_time": 49.12173577863723
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1493.0,
|
|
"completions/max_terminated_length": 1493.0,
|
|
"completions/mean_length": 812.453125,
|
|
"completions/mean_terminated_length": 812.453125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 0.9595177620649338,
|
|
"epoch": 0.7267573696145124,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.008898406696971506,
|
|
"learning_rate": 2.7380952380952385e-07,
|
|
"loss": -0.0043,
|
|
"num_tokens": 75780469.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1282,
|
|
"step_time": 52.93082863371819
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1353.0,
|
|
"completions/max_terminated_length": 1353.0,
|
|
"completions/mean_length": 734.046875,
|
|
"completions/mean_terminated_length": 734.046875,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.6298907436430454,
|
|
"epoch": 0.7273242630385488,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"kl": 0.00913073550327681,
|
|
"learning_rate": 2.732426303854875e-07,
|
|
"loss": 0.029,
|
|
"num_tokens": 75837592.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1283,
|
|
"step_time": 47.84131388925016
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1405.0,
|
|
"completions/max_terminated_length": 1405.0,
|
|
"completions/mean_length": 690.84375,
|
|
"completions/mean_terminated_length": 690.84375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.6134188510477543,
|
|
"epoch": 0.7278911564625851,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.006679923812043853,
|
|
"learning_rate": 2.7267573696145124e-07,
|
|
"loss": 0.0234,
|
|
"num_tokens": 75891998.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1284,
|
|
"step_time": 49.93666685372591
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1356.0,
|
|
"completions/max_terminated_length": 1356.0,
|
|
"completions/mean_length": 762.8125,
|
|
"completions/mean_terminated_length": 762.8125,
|
|
"completions/min_length": 342.0,
|
|
"completions/min_terminated_length": 342.0,
|
|
"entropy": 0.925384433940053,
|
|
"epoch": 0.7284580498866213,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.014122795866569504,
|
|
"learning_rate": 2.72108843537415e-07,
|
|
"loss": 0.0126,
|
|
"num_tokens": 75950530.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1285,
|
|
"step_time": 47.307156290858984
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1639.0,
|
|
"completions/max_terminated_length": 1639.0,
|
|
"completions/mean_length": 748.921875,
|
|
"completions/mean_terminated_length": 748.921875,
|
|
"completions/min_length": 378.0,
|
|
"completions/min_terminated_length": 378.0,
|
|
"entropy": 0.73979701846838,
|
|
"epoch": 0.7290249433106576,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"kl": 0.014161539409542456,
|
|
"learning_rate": 2.715419501133787e-07,
|
|
"loss": 0.0106,
|
|
"num_tokens": 76010269.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1286,
|
|
"step_time": 58.487636857666075
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1461.0,
|
|
"completions/max_terminated_length": 1461.0,
|
|
"completions/mean_length": 758.140625,
|
|
"completions/mean_terminated_length": 758.140625,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.8998379223048687,
|
|
"epoch": 0.7295918367346939,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3203125,
|
|
"kl": 0.0072586629539728165,
|
|
"learning_rate": 2.709750566893424e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 76069590.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1287,
|
|
"step_time": 51.004737731069326
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1433.0,
|
|
"completions/max_terminated_length": 1433.0,
|
|
"completions/mean_length": 724.0625,
|
|
"completions/mean_terminated_length": 724.0625,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.9344060737639666,
|
|
"epoch": 0.7301587301587301,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.62109375,
|
|
"kl": 0.006053238495951518,
|
|
"learning_rate": 2.7040816326530613e-07,
|
|
"loss": -0.0054,
|
|
"num_tokens": 76125466.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7531349658966064,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1288,
|
|
"step_time": 50.324384422041476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1427.0,
|
|
"completions/max_terminated_length": 1427.0,
|
|
"completions/mean_length": 765.4375,
|
|
"completions/mean_terminated_length": 765.4375,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.7832475490868092,
|
|
"epoch": 0.7307256235827665,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.61328125,
|
|
"kl": 0.006733076588716358,
|
|
"learning_rate": 2.698412698412698e-07,
|
|
"loss": 0.017,
|
|
"num_tokens": 76183430.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1289,
|
|
"step_time": 50.002171056345105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1171.0,
|
|
"completions/max_terminated_length": 1171.0,
|
|
"completions/mean_length": 753.546875,
|
|
"completions/mean_terminated_length": 753.546875,
|
|
"completions/min_length": 316.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"entropy": 0.9394499845802784,
|
|
"epoch": 0.7312925170068028,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.008889567703590728,
|
|
"learning_rate": 2.692743764172336e-07,
|
|
"loss": 0.0143,
|
|
"num_tokens": 76242353.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1290,
|
|
"step_time": 41.52156690042466
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1258.0,
|
|
"completions/max_terminated_length": 1258.0,
|
|
"completions/mean_length": 759.5,
|
|
"completions/mean_terminated_length": 759.5,
|
|
"completions/min_length": 393.0,
|
|
"completions/min_terminated_length": 393.0,
|
|
"entropy": 0.7363346572965384,
|
|
"epoch": 0.731859410430839,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.011196116494829766,
|
|
"learning_rate": 2.6870748299319727e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 76302841.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.653468668460846,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1291,
|
|
"step_time": 45.646698226220906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1260.0,
|
|
"completions/max_terminated_length": 1260.0,
|
|
"completions/mean_length": 729.921875,
|
|
"completions/mean_terminated_length": 729.921875,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.8937126435339451,
|
|
"epoch": 0.7324263038548753,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.396484375,
|
|
"kl": 0.007445986469974741,
|
|
"learning_rate": 2.6814058956916097e-07,
|
|
"loss": 0.0208,
|
|
"num_tokens": 76359164.0,
|
|
"reward": 0.5234375,
|
|
"reward_std": 0.9103884100914001,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1292,
|
|
"step_time": 44.65016115922481
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1571.0,
|
|
"completions/max_terminated_length": 1571.0,
|
|
"completions/mean_length": 763.015625,
|
|
"completions/mean_terminated_length": 763.015625,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.901191134005785,
|
|
"epoch": 0.7329931972789115,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"kl": 0.007380922048469074,
|
|
"learning_rate": 2.675736961451247e-07,
|
|
"loss": 0.0079,
|
|
"num_tokens": 76419261.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1293,
|
|
"step_time": 55.57816562987864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1938.0,
|
|
"completions/max_terminated_length": 1938.0,
|
|
"completions/mean_length": 791.3125,
|
|
"completions/mean_terminated_length": 791.3125,
|
|
"completions/min_length": 206.0,
|
|
"completions/min_terminated_length": 206.0,
|
|
"entropy": 0.9699136801064014,
|
|
"epoch": 0.7335600907029478,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.017291870375629514,
|
|
"learning_rate": 2.6700680272108847e-07,
|
|
"loss": 0.0131,
|
|
"num_tokens": 76481481.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1294,
|
|
"step_time": 69.27341264113784
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1885.0,
|
|
"completions/max_terminated_length": 1885.0,
|
|
"completions/mean_length": 819.828125,
|
|
"completions/mean_terminated_length": 819.828125,
|
|
"completions/min_length": 427.0,
|
|
"completions/min_terminated_length": 427.0,
|
|
"entropy": 0.813483277335763,
|
|
"epoch": 0.7341269841269841,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.006324895774014294,
|
|
"learning_rate": 2.664399092970521e-07,
|
|
"loss": 0.0201,
|
|
"num_tokens": 76544014.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1295,
|
|
"step_time": 66.80181924719363
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1483.0,
|
|
"completions/max_terminated_length": 1483.0,
|
|
"completions/mean_length": 820.765625,
|
|
"completions/mean_terminated_length": 820.765625,
|
|
"completions/min_length": 462.0,
|
|
"completions/min_terminated_length": 462.0,
|
|
"entropy": 0.747387794777751,
|
|
"epoch": 0.7346938775510204,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.400390625,
|
|
"kl": 0.010703829815611243,
|
|
"learning_rate": 2.6587301587301586e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 76605727.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1296,
|
|
"step_time": 52.425307758152485
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1623.0,
|
|
"completions/max_terminated_length": 1623.0,
|
|
"completions/mean_length": 768.921875,
|
|
"completions/mean_terminated_length": 768.921875,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.8974382765591145,
|
|
"epoch": 0.7352607709750567,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.007383863718132488,
|
|
"learning_rate": 2.653061224489796e-07,
|
|
"loss": 0.0124,
|
|
"num_tokens": 76666370.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1297,
|
|
"step_time": 58.10066665709019
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1332.0,
|
|
"completions/max_terminated_length": 1332.0,
|
|
"completions/mean_length": 761.09375,
|
|
"completions/mean_terminated_length": 761.09375,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.9388503842055798,
|
|
"epoch": 0.735827664399093,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"kl": 0.010892568621784449,
|
|
"learning_rate": 2.647392290249433e-07,
|
|
"loss": -0.0109,
|
|
"num_tokens": 76726752.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1298,
|
|
"step_time": 47.77322860341519
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1656.0,
|
|
"completions/max_terminated_length": 1656.0,
|
|
"completions/mean_length": 761.78125,
|
|
"completions/mean_terminated_length": 761.78125,
|
|
"completions/min_length": 21.0,
|
|
"completions/min_terminated_length": 21.0,
|
|
"entropy": 0.816766981035471,
|
|
"epoch": 0.7363945578231292,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.007609797830809839,
|
|
"learning_rate": 2.64172335600907e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 76784842.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5395448207855225,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1299,
|
|
"step_time": 58.15237878728658
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1756.0,
|
|
"completions/max_terminated_length": 1756.0,
|
|
"completions/mean_length": 777.140625,
|
|
"completions/mean_terminated_length": 777.140625,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.8178585413843393,
|
|
"epoch": 0.7369614512471655,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.009846978209679946,
|
|
"learning_rate": 2.6360544217687075e-07,
|
|
"loss": -0.0335,
|
|
"num_tokens": 76846043.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1300,
|
|
"step_time": 62.135099502280354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1457.0,
|
|
"completions/max_terminated_length": 1457.0,
|
|
"completions/mean_length": 813.234375,
|
|
"completions/mean_terminated_length": 813.234375,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.6848905384540558,
|
|
"epoch": 0.7375283446712018,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.006538849964272231,
|
|
"learning_rate": 2.6303854875283444e-07,
|
|
"loss": -0.002,
|
|
"num_tokens": 76907810.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.852168083190918,
|
|
"rewards/accuracy_reward_func/mean": 0.5,
|
|
"rewards/accuracy_reward_func/std": 0.8728715777397156,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1301,
|
|
"step_time": 51.57597290817648
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1447.0,
|
|
"completions/max_terminated_length": 1447.0,
|
|
"completions/mean_length": 810.640625,
|
|
"completions/mean_terminated_length": 810.640625,
|
|
"completions/min_length": 242.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"entropy": 0.6645511239767075,
|
|
"epoch": 0.7380952380952381,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.396484375,
|
|
"kl": 0.005681007693056017,
|
|
"learning_rate": 2.624716553287982e-07,
|
|
"loss": 0.0098,
|
|
"num_tokens": 76969587.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1302,
|
|
"step_time": 50.856564400717616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1035.0,
|
|
"completions/max_terminated_length": 1035.0,
|
|
"completions/mean_length": 620.78125,
|
|
"completions/mean_terminated_length": 620.78125,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.6836046054959297,
|
|
"epoch": 0.7386621315192744,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.014384060545125976,
|
|
"learning_rate": 2.619047619047619e-07,
|
|
"loss": 0.0234,
|
|
"num_tokens": 77020325.0,
|
|
"reward": 0.546875,
|
|
"reward_std": 0.9288957118988037,
|
|
"rewards/accuracy_reward_func/mean": 0.5,
|
|
"rewards/accuracy_reward_func/std": 0.8728715777397156,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1303,
|
|
"step_time": 37.2075195973739
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1968.0,
|
|
"completions/max_terminated_length": 1968.0,
|
|
"completions/mean_length": 821.015625,
|
|
"completions/mean_terminated_length": 821.015625,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 0.7058903947472572,
|
|
"epoch": 0.7392290249433107,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48828125,
|
|
"kl": 0.009722277813125402,
|
|
"learning_rate": 2.613378684807256e-07,
|
|
"loss": 0.0018,
|
|
"num_tokens": 77083134.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 1304,
|
|
"step_time": 69.87466052174568
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2007.0,
|
|
"completions/max_terminated_length": 2007.0,
|
|
"completions/mean_length": 952.28125,
|
|
"completions/mean_terminated_length": 952.28125,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.6597837656736374,
|
|
"epoch": 0.7397959183673469,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.373046875,
|
|
"kl": 0.007097126770531759,
|
|
"learning_rate": 2.6077097505668934e-07,
|
|
"loss": -0.0154,
|
|
"num_tokens": 77155856.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1305,
|
|
"step_time": 72.48270978685468
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1872.0,
|
|
"completions/max_terminated_length": 1872.0,
|
|
"completions/mean_length": 822.109375,
|
|
"completions/mean_terminated_length": 822.109375,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.6589792240411043,
|
|
"epoch": 0.7403628117913832,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.015984905578079633,
|
|
"learning_rate": 2.602040816326531e-07,
|
|
"loss": -0.0143,
|
|
"num_tokens": 77219759.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1306,
|
|
"step_time": 66.50180557649583
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2061.0,
|
|
"completions/max_terminated_length": 2061.0,
|
|
"completions/mean_length": 794.84375,
|
|
"completions/mean_terminated_length": 794.84375,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.7259137369692326,
|
|
"epoch": 0.7409297052154195,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.189453125,
|
|
"kl": 0.012303568684728816,
|
|
"learning_rate": 2.5963718820861673e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 77280781.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1307,
|
|
"step_time": 73.14075470529497
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1889.0,
|
|
"completions/max_terminated_length": 1889.0,
|
|
"completions/mean_length": 681.3125,
|
|
"completions/mean_terminated_length": 681.3125,
|
|
"completions/min_length": 225.0,
|
|
"completions/min_terminated_length": 225.0,
|
|
"entropy": 0.792594239115715,
|
|
"epoch": 0.7414965986394558,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.006777283866540529,
|
|
"learning_rate": 2.590702947845805e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 77334401.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6172134280204773,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1308,
|
|
"step_time": 67.00924664177
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1968.0,
|
|
"completions/max_terminated_length": 1968.0,
|
|
"completions/mean_length": 768.734375,
|
|
"completions/mean_terminated_length": 768.734375,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.5566640328615904,
|
|
"epoch": 0.7420634920634921,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.014018147820024751,
|
|
"learning_rate": 2.5850340136054423e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 77393992.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1309,
|
|
"step_time": 70.7074468974024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1762.0,
|
|
"completions/max_terminated_length": 1762.0,
|
|
"completions/mean_length": 843.578125,
|
|
"completions/mean_terminated_length": 843.578125,
|
|
"completions/min_length": 423.0,
|
|
"completions/min_terminated_length": 423.0,
|
|
"entropy": 0.6671493239700794,
|
|
"epoch": 0.7426303854875284,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.47265625,
|
|
"kl": 0.020442446621018462,
|
|
"learning_rate": 2.57936507936508e-07,
|
|
"loss": 0.0297,
|
|
"num_tokens": 77460093.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1310,
|
|
"step_time": 63.45249472092837
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1588.0,
|
|
"completions/max_terminated_length": 1588.0,
|
|
"completions/mean_length": 728.5625,
|
|
"completions/mean_terminated_length": 728.5625,
|
|
"completions/min_length": 259.0,
|
|
"completions/min_terminated_length": 259.0,
|
|
"entropy": 0.7086256965994835,
|
|
"epoch": 0.7431972789115646,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.609375,
|
|
"kl": 0.019253130041761324,
|
|
"learning_rate": 2.573696145124716e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 77518329.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1311,
|
|
"step_time": 56.265187757089734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1811.0,
|
|
"completions/max_terminated_length": 1811.0,
|
|
"completions/mean_length": 767.296875,
|
|
"completions/mean_terminated_length": 767.296875,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.7795978803187609,
|
|
"epoch": 0.7437641723356009,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6796875,
|
|
"kl": 0.00936562375864014,
|
|
"learning_rate": 2.5680272108843537e-07,
|
|
"loss": -0.0028,
|
|
"num_tokens": 77577428.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1312,
|
|
"step_time": 63.59273668937385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1490.0,
|
|
"completions/max_terminated_length": 1490.0,
|
|
"completions/mean_length": 932.625,
|
|
"completions/mean_terminated_length": 932.625,
|
|
"completions/min_length": 477.0,
|
|
"completions/min_terminated_length": 477.0,
|
|
"entropy": 0.5991004668176174,
|
|
"epoch": 0.7443310657596371,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.0050917043699882925,
|
|
"learning_rate": 2.562358276643991e-07,
|
|
"loss": -0.0121,
|
|
"num_tokens": 77646468.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1313,
|
|
"step_time": 52.13757051061839
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1682.0,
|
|
"completions/max_terminated_length": 1682.0,
|
|
"completions/mean_length": 801.984375,
|
|
"completions/mean_terminated_length": 801.984375,
|
|
"completions/min_length": 195.0,
|
|
"completions/min_terminated_length": 195.0,
|
|
"entropy": 0.7486903565004468,
|
|
"epoch": 0.7448979591836735,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.490234375,
|
|
"kl": 0.008225985919125378,
|
|
"learning_rate": 2.556689342403628e-07,
|
|
"loss": 0.0292,
|
|
"num_tokens": 77709835.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.8742556571960449,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1314,
|
|
"step_time": 60.640764138661325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1466.0,
|
|
"completions/max_terminated_length": 1466.0,
|
|
"completions/mean_length": 832.53125,
|
|
"completions/mean_terminated_length": 832.53125,
|
|
"completions/min_length": 274.0,
|
|
"completions/min_terminated_length": 274.0,
|
|
"entropy": 0.7583224773406982,
|
|
"epoch": 0.7454648526077098,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.00685890564636793,
|
|
"learning_rate": 2.551020408163265e-07,
|
|
"loss": 0.0203,
|
|
"num_tokens": 77774469.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1315,
|
|
"step_time": 51.66778673231602
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1852.0,
|
|
"completions/max_terminated_length": 1852.0,
|
|
"completions/mean_length": 783.71875,
|
|
"completions/mean_terminated_length": 783.71875,
|
|
"completions/min_length": 391.0,
|
|
"completions/min_terminated_length": 391.0,
|
|
"entropy": 0.6801396571099758,
|
|
"epoch": 0.746031746031746,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.013807280804030597,
|
|
"learning_rate": 2.5453514739229026e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 77834547.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7175520658493042,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.09375,
|
|
"rewards/format_reward_func/std": 0.19669894874095917,
|
|
"step": 1316,
|
|
"step_time": 65.5430804071948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1089.0,
|
|
"completions/mean_length": 725.609375,
|
|
"completions/mean_terminated_length": 672.1111450195312,
|
|
"completions/min_length": 230.0,
|
|
"completions/min_terminated_length": 230.0,
|
|
"entropy": 0.9323985762894154,
|
|
"epoch": 0.7465986394557823,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.494140625,
|
|
"kl": 0.013572649244451895,
|
|
"learning_rate": 2.5396825396825396e-07,
|
|
"loss": -0.0109,
|
|
"num_tokens": 77897682.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6664806604385376,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1317,
|
|
"step_time": 178.16909020859748
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1441.0,
|
|
"completions/max_terminated_length": 1441.0,
|
|
"completions/mean_length": 843.359375,
|
|
"completions/mean_terminated_length": 843.359375,
|
|
"completions/min_length": 182.0,
|
|
"completions/min_terminated_length": 182.0,
|
|
"entropy": 0.7985265720635653,
|
|
"epoch": 0.7471655328798186,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3515625,
|
|
"kl": 0.007031826724414714,
|
|
"learning_rate": 2.534013605442177e-07,
|
|
"loss": -0.0296,
|
|
"num_tokens": 77960609.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 1318,
|
|
"step_time": 51.01844102144241
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1154.0,
|
|
"completions/max_terminated_length": 1154.0,
|
|
"completions/mean_length": 802.1875,
|
|
"completions/mean_terminated_length": 802.1875,
|
|
"completions/min_length": 441.0,
|
|
"completions/min_terminated_length": 441.0,
|
|
"entropy": 0.761490236967802,
|
|
"epoch": 0.7477324263038548,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5703125,
|
|
"kl": 0.0070764376723673195,
|
|
"learning_rate": 2.528344671201814e-07,
|
|
"loss": 0.0047,
|
|
"num_tokens": 78022253.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7175520658493042,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1319,
|
|
"step_time": 41.24426679685712
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1182.0,
|
|
"completions/max_terminated_length": 1182.0,
|
|
"completions/mean_length": 731.78125,
|
|
"completions/mean_terminated_length": 731.78125,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.8387901466339827,
|
|
"epoch": 0.7482993197278912,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.365234375,
|
|
"kl": 0.007836662465706468,
|
|
"learning_rate": 2.522675736961451e-07,
|
|
"loss": 0.0085,
|
|
"num_tokens": 78079615.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 1320,
|
|
"step_time": 41.96160435024649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1717.0,
|
|
"completions/max_terminated_length": 1717.0,
|
|
"completions/mean_length": 664.390625,
|
|
"completions/mean_terminated_length": 664.390625,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 1.1970753818750381,
|
|
"epoch": 0.7488662131519275,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55078125,
|
|
"kl": 0.012836450187023729,
|
|
"learning_rate": 2.5170068027210885e-07,
|
|
"loss": -0.0219,
|
|
"num_tokens": 78137144.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1321,
|
|
"step_time": 63.000695964321494
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1416.0,
|
|
"completions/max_terminated_length": 1416.0,
|
|
"completions/mean_length": 782.828125,
|
|
"completions/mean_terminated_length": 782.828125,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.6070452481508255,
|
|
"epoch": 0.7494331065759637,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"kl": 0.008763950900174677,
|
|
"learning_rate": 2.511337868480726e-07,
|
|
"loss": -0.0078,
|
|
"num_tokens": 78196357.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 1322,
|
|
"step_time": 49.674625519663095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1534.0,
|
|
"completions/max_terminated_length": 1534.0,
|
|
"completions/mean_length": 771.015625,
|
|
"completions/mean_terminated_length": 771.015625,
|
|
"completions/min_length": 247.0,
|
|
"completions/min_terminated_length": 247.0,
|
|
"entropy": 0.7325721830129623,
|
|
"epoch": 0.75,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"kl": 0.021216300054220483,
|
|
"learning_rate": 2.5056689342403624e-07,
|
|
"loss": -0.0051,
|
|
"num_tokens": 78257446.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 1323,
|
|
"step_time": 55.45931248925626
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 1764,
|
|
"num_input_tokens_seen": 78257446,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|