f343771ea4
Model: azalahmadkhan/Llama-3.2-3B-Instruct-GRPO-vanilla-G-8-novllm-25pct Source: Original Platform
13265 lines
486 KiB
JSON
13265 lines
486 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.25,
|
|
"eval_steps": 500,
|
|
"global_step": 441,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3204.0,
|
|
"completions/mean_length": 1194.96875,
|
|
"completions/mean_terminated_length": 894.862060546875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.504598390311003,
|
|
"epoch": 0.0005668934240362812,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.189453125,
|
|
"kl": 0.000330764095451741,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0204,
|
|
"num_tokens": 86206.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.40999117493629456,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 1,
|
|
"step_time": 290.4004911398515
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3540.0,
|
|
"completions/mean_length": 1322.1875,
|
|
"completions/mean_terminated_length": 746.4906005859375,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.345731921494007,
|
|
"epoch": 0.0011337868480725624,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.2314453125,
|
|
"kl": 0.0003442415154495393,
|
|
"learning_rate": 9.994331065759637e-07,
|
|
"loss": 0.0463,
|
|
"num_tokens": 183818.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 2,
|
|
"step_time": 303.91933149658144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.234375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3854.0,
|
|
"completions/mean_length": 2008.53125,
|
|
"completions/mean_terminated_length": 1369.5101318359375,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 5.243577856570482,
|
|
"epoch": 0.0017006802721088435,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.10302734375,
|
|
"kl": 0.00029097342212480726,
|
|
"learning_rate": 9.988662131519275e-07,
|
|
"loss": 0.0205,
|
|
"num_tokens": 323068.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 3,
|
|
"step_time": 299.3002619827166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3719.0,
|
|
"completions/mean_length": 1336.40625,
|
|
"completions/mean_terminated_length": 699.5769653320312,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.6891971342265606,
|
|
"epoch": 0.0022675736961451248,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2158203125,
|
|
"kl": 0.00041494359993521357,
|
|
"learning_rate": 9.982993197278912e-07,
|
|
"loss": 0.0298,
|
|
"num_tokens": 420150.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 4,
|
|
"step_time": 294.2055949298665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3626.0,
|
|
"completions/mean_length": 1425.28125,
|
|
"completions/mean_terminated_length": 930.7037353515625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.3760005459189415,
|
|
"epoch": 0.002834467120181406,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1953125,
|
|
"kl": 0.0003999843729616259,
|
|
"learning_rate": 9.977324263038547e-07,
|
|
"loss": 0.0157,
|
|
"num_tokens": 522424.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 5,
|
|
"step_time": 293.4846060844138
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3887.0,
|
|
"completions/mean_length": 1280.796875,
|
|
"completions/mean_terminated_length": 989.5689697265625,
|
|
"completions/min_length": 82.0,
|
|
"completions/min_terminated_length": 82.0,
|
|
"entropy": 4.2538034208118916,
|
|
"epoch": 0.003401360544217687,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 6.29425048828125e-05,
|
|
"kl": 0.0003539047747835866,
|
|
"learning_rate": 9.971655328798185e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 616499.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 6,
|
|
"step_time": 296.5241157421842
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3576.0,
|
|
"completions/mean_length": 1286.109375,
|
|
"completions/mean_terminated_length": 765.75927734375,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.3659513257443905,
|
|
"epoch": 0.003968253968253968,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2255859375,
|
|
"kl": 0.0003789493484873674,
|
|
"learning_rate": 9.965986394557822e-07,
|
|
"loss": 0.0023,
|
|
"num_tokens": 712794.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.7449815273284912,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 7,
|
|
"step_time": 309.2328038820997
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3994.0,
|
|
"completions/mean_length": 1192.53125,
|
|
"completions/mean_terminated_length": 654.8518676757812,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 1.9319973811507225,
|
|
"epoch": 0.0045351473922902496,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.0004903619992546737,
|
|
"learning_rate": 9.96031746031746e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 802204.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 8,
|
|
"step_time": 297.1033435128629
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3889.0,
|
|
"completions/mean_length": 1329.078125,
|
|
"completions/mean_terminated_length": 754.8113403320312,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 2.8315951824188232,
|
|
"epoch": 0.00510204081632653,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1494140625,
|
|
"kl": 0.00039701382593193557,
|
|
"learning_rate": 9.954648526077097e-07,
|
|
"loss": -0.0048,
|
|
"num_tokens": 899905.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5866720080375671,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 9,
|
|
"step_time": 299.20237035956234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3610.0,
|
|
"completions/mean_length": 1129.09375,
|
|
"completions/mean_terminated_length": 764.7368774414062,
|
|
"completions/min_length": 89.0,
|
|
"completions/min_terminated_length": 89.0,
|
|
"entropy": 2.454593241214752,
|
|
"epoch": 0.005668934240362812,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.00046382144591916585,
|
|
"learning_rate": 9.948979591836735e-07,
|
|
"loss": -0.0261,
|
|
"num_tokens": 982783.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 10,
|
|
"step_time": 295.87636501993984
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3606.0,
|
|
"completions/mean_length": 1484.53125,
|
|
"completions/mean_terminated_length": 1000.9259033203125,
|
|
"completions/min_length": 109.0,
|
|
"completions/min_terminated_length": 109.0,
|
|
"entropy": 4.456293493509293,
|
|
"epoch": 0.006235827664399093,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1416015625,
|
|
"kl": 0.0003735653808689676,
|
|
"learning_rate": 9.94331065759637e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 1088801.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 11,
|
|
"step_time": 296.7332943305373
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3438.0,
|
|
"completions/mean_length": 1243.078125,
|
|
"completions/mean_terminated_length": 892.7192993164062,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"entropy": 3.2929567843675613,
|
|
"epoch": 0.006802721088435374,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2041015625,
|
|
"kl": 0.0003951908856834052,
|
|
"learning_rate": 9.937641723356008e-07,
|
|
"loss": 0.0524,
|
|
"num_tokens": 1179622.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 12,
|
|
"step_time": 294.44508679863065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3944.0,
|
|
"completions/mean_length": 1431.640625,
|
|
"completions/mean_terminated_length": 995.654541015625,
|
|
"completions/min_length": 122.0,
|
|
"completions/min_terminated_length": 122.0,
|
|
"entropy": 3.8930897265672684,
|
|
"epoch": 0.007369614512471655,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2265625,
|
|
"kl": 0.00038424173817475094,
|
|
"learning_rate": 9.931972789115645e-07,
|
|
"loss": -0.0218,
|
|
"num_tokens": 1283623.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 13,
|
|
"step_time": 294.92571747582406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4029.0,
|
|
"completions/mean_length": 1600.640625,
|
|
"completions/mean_terminated_length": 964.5686645507812,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.1041195038706064,
|
|
"epoch": 0.007936507936507936,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.11669921875,
|
|
"kl": 0.00040818309116730234,
|
|
"learning_rate": 9.926303854875283e-07,
|
|
"loss": -0.0074,
|
|
"num_tokens": 1398664.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 14,
|
|
"step_time": 297.8464042544365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3815.0,
|
|
"completions/mean_length": 1455.21875,
|
|
"completions/mean_terminated_length": 845.8077392578125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.581351801753044,
|
|
"epoch": 0.008503401360544218,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1083984375,
|
|
"kl": 0.0003867176119456417,
|
|
"learning_rate": 9.92063492063492e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 1503838.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5833333730697632,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 15,
|
|
"step_time": 294.84605468437076
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3655.0,
|
|
"completions/mean_length": 1439.03125,
|
|
"completions/mean_terminated_length": 887.5848999023438,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.61439785733819,
|
|
"epoch": 0.009070294784580499,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1962890625,
|
|
"kl": 0.00041348650029249256,
|
|
"learning_rate": 9.914965986394558e-07,
|
|
"loss": 0.028,
|
|
"num_tokens": 1607136.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 16,
|
|
"step_time": 295.82693486101925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3806.0,
|
|
"completions/mean_length": 1300.1875,
|
|
"completions/mean_terminated_length": 782.4444580078125,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"entropy": 3.3572088442742825,
|
|
"epoch": 0.00963718820861678,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.16015625,
|
|
"kl": 0.00045007884546066634,
|
|
"learning_rate": 9.909297052154193e-07,
|
|
"loss": 0.011,
|
|
"num_tokens": 1702620.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 17,
|
|
"step_time": 294.88669333048165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3298.0,
|
|
"completions/mean_length": 1205.390625,
|
|
"completions/mean_terminated_length": 850.4035034179688,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 3.502040285617113,
|
|
"epoch": 0.01020408163265306,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 7.43865966796875e-05,
|
|
"kl": 0.0004136692868996761,
|
|
"learning_rate": 9.90362811791383e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1790741.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 18,
|
|
"step_time": 295.91272540576756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3705.0,
|
|
"completions/mean_length": 978.734375,
|
|
"completions/mean_terminated_length": 714.559326171875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.546327754855156,
|
|
"epoch": 0.010770975056689343,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.32421875,
|
|
"kl": 0.0004484470018724096,
|
|
"learning_rate": 9.897959183673468e-07,
|
|
"loss": 0.0622,
|
|
"num_tokens": 1864268.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 19,
|
|
"step_time": 292.4661794034764
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3801.0,
|
|
"completions/mean_length": 1386.65625,
|
|
"completions/mean_terminated_length": 999.607177734375,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.70910806581378,
|
|
"epoch": 0.011337868480725623,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.00038008083629392786,
|
|
"learning_rate": 9.892290249433106e-07,
|
|
"loss": 0.0765,
|
|
"num_tokens": 1964438.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 20,
|
|
"step_time": 301.6167815560475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3725.0,
|
|
"completions/mean_length": 1592.34375,
|
|
"completions/mean_terminated_length": 954.1569213867188,
|
|
"completions/min_length": 89.0,
|
|
"completions/min_terminated_length": 89.0,
|
|
"entropy": 3.0310422237962484,
|
|
"epoch": 0.011904761904761904,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1845703125,
|
|
"kl": 0.00045945892634335905,
|
|
"learning_rate": 9.886621315192743e-07,
|
|
"loss": -0.0155,
|
|
"num_tokens": 2080588.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 21,
|
|
"step_time": 304.0985494265333
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3757.0,
|
|
"completions/mean_length": 1292.734375,
|
|
"completions/mean_terminated_length": 773.6111450195312,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.643630724400282,
|
|
"epoch": 0.012471655328798186,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.00043581062300290796,
|
|
"learning_rate": 9.88095238095238e-07,
|
|
"loss": 0.0503,
|
|
"num_tokens": 2183251.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5246290564537048,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 22,
|
|
"step_time": 343.65668355021626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3839.0,
|
|
"completions/mean_length": 1119.25,
|
|
"completions/mean_terminated_length": 811.3103637695312,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.981692060828209,
|
|
"epoch": 0.013038548752834467,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.18359375,
|
|
"kl": 0.0004370460965219536,
|
|
"learning_rate": 9.875283446712018e-07,
|
|
"loss": -0.0105,
|
|
"num_tokens": 2267099.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.41277948021888733,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 23,
|
|
"step_time": 296.2689810972661
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3834.0,
|
|
"completions/mean_length": 1517.109375,
|
|
"completions/mean_terminated_length": 1095.1090087890625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.816467195749283,
|
|
"epoch": 0.013605442176870748,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1767578125,
|
|
"kl": 0.00040354001430387143,
|
|
"learning_rate": 9.869614512471656e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 2376002.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.5525464415550232,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 24,
|
|
"step_time": 295.8765085712075
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3787.0,
|
|
"completions/mean_length": 1134.984375,
|
|
"completions/mean_terminated_length": 884.0508422851562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.6541102938354015,
|
|
"epoch": 0.01417233560090703,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.0004305867323637358,
|
|
"learning_rate": 9.86394557823129e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 2460153.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 25,
|
|
"step_time": 298.3754459451884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4090.0,
|
|
"completions/mean_length": 1289.859375,
|
|
"completions/mean_terminated_length": 945.24560546875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.4574208334088326,
|
|
"epoch": 0.01473922902494331,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.0004790655384567799,
|
|
"learning_rate": 9.858276643990928e-07,
|
|
"loss": -0.0492,
|
|
"num_tokens": 2553296.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 26,
|
|
"step_time": 293.37725717946887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3986.0,
|
|
"completions/mean_length": 1333.1875,
|
|
"completions/mean_terminated_length": 993.8947143554688,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.69367503374815,
|
|
"epoch": 0.015306122448979591,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.275390625,
|
|
"kl": 0.0004016220236735535,
|
|
"learning_rate": 9.852607709750566e-07,
|
|
"loss": 0.0141,
|
|
"num_tokens": 2650476.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.46691107749938965,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 27,
|
|
"step_time": 294.41278407908976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3878.0,
|
|
"completions/mean_length": 1388.0,
|
|
"completions/mean_terminated_length": 825.9622802734375,
|
|
"completions/min_length": 115.0,
|
|
"completions/min_terminated_length": 115.0,
|
|
"entropy": 4.408259846270084,
|
|
"epoch": 0.015873015873015872,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.212890625,
|
|
"kl": 0.0004581758648782852,
|
|
"learning_rate": 9.846938775510203e-07,
|
|
"loss": 0.0412,
|
|
"num_tokens": 2750436.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 28,
|
|
"step_time": 294.78673951141536
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3728.0,
|
|
"completions/mean_length": 1695.734375,
|
|
"completions/mean_terminated_length": 1197.5660400390625,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 3.804402470588684,
|
|
"epoch": 0.016439909297052153,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.00035922019833378727,
|
|
"learning_rate": 9.84126984126984e-07,
|
|
"loss": -0.001,
|
|
"num_tokens": 2873219.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 29,
|
|
"step_time": 300.3509062519297
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4084.0,
|
|
"completions/mean_length": 1472.375,
|
|
"completions/mean_terminated_length": 866.923095703125,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"entropy": 2.803918108344078,
|
|
"epoch": 0.017006802721088437,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1962890625,
|
|
"kl": 0.0004838097711399314,
|
|
"learning_rate": 9.835600907029478e-07,
|
|
"loss": 0.0102,
|
|
"num_tokens": 2980107.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 30,
|
|
"step_time": 306.3423152305186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4080.0,
|
|
"completions/mean_length": 1367.78125,
|
|
"completions/mean_terminated_length": 862.5555419921875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.843935079872608,
|
|
"epoch": 0.017573696145124718,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.15234375,
|
|
"kl": 0.00045960527859278955,
|
|
"learning_rate": 9.829931972789116e-07,
|
|
"loss": 0.0181,
|
|
"num_tokens": 3078581.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 31,
|
|
"step_time": 295.59960942994803
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3777.0,
|
|
"completions/mean_length": 1544.828125,
|
|
"completions/mean_terminated_length": 956.09619140625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.920840870589018,
|
|
"epoch": 0.018140589569160998,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.0004275208129911334,
|
|
"learning_rate": 9.824263038548753e-07,
|
|
"loss": -0.0099,
|
|
"num_tokens": 3191306.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 32,
|
|
"step_time": 305.35615600366145
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3834.0,
|
|
"completions/mean_length": 1434.1875,
|
|
"completions/mean_terminated_length": 688.8800048828125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.467706087976694,
|
|
"epoch": 0.01870748299319728,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.2353515625,
|
|
"kl": 0.0004915133140457328,
|
|
"learning_rate": 9.818594104308389e-07,
|
|
"loss": 0.0415,
|
|
"num_tokens": 3293022.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6664806604385376,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 33,
|
|
"step_time": 294.9509489592165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3522.0,
|
|
"completions/mean_length": 1028.40625,
|
|
"completions/mean_terminated_length": 711.0689697265625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.900582242757082,
|
|
"epoch": 0.01927437641723356,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.236328125,
|
|
"kl": 0.0004916769539704546,
|
|
"learning_rate": 9.812925170068026e-07,
|
|
"loss": 0.0127,
|
|
"num_tokens": 3370272.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 34,
|
|
"step_time": 294.09036393929273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4024.0,
|
|
"completions/mean_length": 1438.984375,
|
|
"completions/mean_terminated_length": 1059.4107666015625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.51833862811327,
|
|
"epoch": 0.01984126984126984,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2138671875,
|
|
"kl": 0.0004253339166098158,
|
|
"learning_rate": 9.807256235827664e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 3475223.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 35,
|
|
"step_time": 302.7437364095822
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3704.0,
|
|
"completions/mean_length": 1525.28125,
|
|
"completions/mean_terminated_length": 932.0385131835938,
|
|
"completions/min_length": 36.0,
|
|
"completions/min_terminated_length": 36.0,
|
|
"entropy": 3.4461216628551483,
|
|
"epoch": 0.02040816326530612,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.142578125,
|
|
"kl": 0.0004800664210051764,
|
|
"learning_rate": 9.801587301587301e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 3584969.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 36,
|
|
"step_time": 298.9434612961486
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4052.0,
|
|
"completions/mean_length": 1216.53125,
|
|
"completions/mean_terminated_length": 862.9122924804688,
|
|
"completions/min_length": 36.0,
|
|
"completions/min_terminated_length": 36.0,
|
|
"entropy": 3.0853926092386246,
|
|
"epoch": 0.020975056689342405,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.0004732625448014005,
|
|
"learning_rate": 9.795918367346939e-07,
|
|
"loss": 0.0305,
|
|
"num_tokens": 3675523.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4917473793029785,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 37,
|
|
"step_time": 303.82939208392054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3573.0,
|
|
"completions/mean_length": 1365.390625,
|
|
"completions/mean_terminated_length": 859.7222290039062,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.5588138587772846,
|
|
"epoch": 0.021541950113378686,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.0005098498504594318,
|
|
"learning_rate": 9.790249433106576e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 3773716.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 38,
|
|
"step_time": 293.0893633402884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3852.0,
|
|
"completions/mean_length": 1856.796875,
|
|
"completions/mean_terminated_length": 1229.8199462890625,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"entropy": 4.590333938598633,
|
|
"epoch": 0.022108843537414966,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1279296875,
|
|
"kl": 0.00041188216528098565,
|
|
"learning_rate": 9.784580498866214e-07,
|
|
"loss": 0.0116,
|
|
"num_tokens": 3904815.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 39,
|
|
"step_time": 296.1281976234168
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4055.0,
|
|
"completions/mean_length": 1368.21875,
|
|
"completions/mean_terminated_length": 1137.0509033203125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 5.263170003890991,
|
|
"epoch": 0.022675736961451247,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2099609375,
|
|
"kl": 0.00041604944817663636,
|
|
"learning_rate": 9.778911564625851e-07,
|
|
"loss": 0.0112,
|
|
"num_tokens": 4008293.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 40,
|
|
"step_time": 314.7039760276675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3685.0,
|
|
"completions/mean_length": 1472.890625,
|
|
"completions/mean_terminated_length": 738.4199829101562,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.3532409109175205,
|
|
"epoch": 0.023242630385487528,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.259765625,
|
|
"kl": 0.0005240712735030684,
|
|
"learning_rate": 9.773242630385487e-07,
|
|
"loss": -0.0396,
|
|
"num_tokens": 4114230.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 41,
|
|
"step_time": 294.0841481247917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4092.0,
|
|
"completions/mean_length": 1695.96875,
|
|
"completions/mean_terminated_length": 1303.236328125,
|
|
"completions/min_length": 231.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"entropy": 4.116798646748066,
|
|
"epoch": 0.023809523809523808,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1650390625,
|
|
"kl": 0.0003809477675531525,
|
|
"learning_rate": 9.767573696145124e-07,
|
|
"loss": -0.0104,
|
|
"num_tokens": 4235436.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 42,
|
|
"step_time": 305.93450375366956
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4051.0,
|
|
"completions/mean_length": 1133.921875,
|
|
"completions/mean_terminated_length": 770.1578979492188,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 3.4529144503176212,
|
|
"epoch": 0.024376417233560092,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.3203125,
|
|
"kl": 0.0005080552914478176,
|
|
"learning_rate": 9.761904761904762e-07,
|
|
"loss": 0.0488,
|
|
"num_tokens": 4319519.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 43,
|
|
"step_time": 294.02621566783637
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3383.0,
|
|
"completions/mean_length": 1556.59375,
|
|
"completions/mean_terminated_length": 970.5769653320312,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 2.7311813980340958,
|
|
"epoch": 0.024943310657596373,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1767578125,
|
|
"kl": 0.00046972488871688256,
|
|
"learning_rate": 9.7562358276644e-07,
|
|
"loss": 0.0289,
|
|
"num_tokens": 4430557.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 44,
|
|
"step_time": 296.04308369010687
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4089.0,
|
|
"completions/mean_length": 1701.234375,
|
|
"completions/mean_terminated_length": 902.9791870117188,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.2801453731954098,
|
|
"epoch": 0.025510204081632654,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.212890625,
|
|
"kl": 0.0005189456205698662,
|
|
"learning_rate": 9.750566893424036e-07,
|
|
"loss": 0.0432,
|
|
"num_tokens": 4552132.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6154024600982666,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 45,
|
|
"step_time": 297.76370439492166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3815.0,
|
|
"completions/mean_length": 1415.671875,
|
|
"completions/mean_terminated_length": 919.3148193359375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.2611497938632965,
|
|
"epoch": 0.026077097505668934,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.138671875,
|
|
"kl": 0.0004771515623360756,
|
|
"learning_rate": 9.744897959183674e-07,
|
|
"loss": 0.0514,
|
|
"num_tokens": 4654695.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 46,
|
|
"step_time": 297.30547140631825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.234375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3786.0,
|
|
"completions/mean_length": 1615.46875,
|
|
"completions/mean_terminated_length": 856.1224365234375,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 2.364711243659258,
|
|
"epoch": 0.026643990929705215,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.1806640625,
|
|
"kl": 0.0005641347979690181,
|
|
"learning_rate": 9.739229024943311e-07,
|
|
"loss": 0.0202,
|
|
"num_tokens": 4772365.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.46922925114631653,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 47,
|
|
"step_time": 309.7107340320945
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4046.0,
|
|
"completions/mean_length": 1357.828125,
|
|
"completions/mean_terminated_length": 1021.5614013671875,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.842697562649846,
|
|
"epoch": 0.027210884353741496,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.0004335203693699441,
|
|
"learning_rate": 9.733560090702947e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 4869858.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 48,
|
|
"step_time": 293.2417637016624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3148.0,
|
|
"completions/mean_length": 1485.171875,
|
|
"completions/mean_terminated_length": 1001.6851806640625,
|
|
"completions/min_length": 23.0,
|
|
"completions/min_terminated_length": 23.0,
|
|
"entropy": 4.688731737434864,
|
|
"epoch": 0.027777777777777776,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1416015625,
|
|
"kl": 0.0004430160397532745,
|
|
"learning_rate": 9.727891156462584e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 4975557.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 49,
|
|
"step_time": 295.9001120068133
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.234375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3707.0,
|
|
"completions/mean_length": 1632.328125,
|
|
"completions/mean_terminated_length": 878.142822265625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.1174243800342083,
|
|
"epoch": 0.02834467120181406,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1318359375,
|
|
"kl": 0.00046842659867252223,
|
|
"learning_rate": 9.722222222222222e-07,
|
|
"loss": -0.0031,
|
|
"num_tokens": 5091690.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 50,
|
|
"step_time": 294.6722645731643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4058.0,
|
|
"completions/mean_length": 1200.78125,
|
|
"completions/mean_terminated_length": 901.27587890625,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 4.219451703131199,
|
|
"epoch": 0.02891156462585034,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2158203125,
|
|
"kl": 0.00045792363744112663,
|
|
"learning_rate": 9.71655328798186e-07,
|
|
"loss": 0.0414,
|
|
"num_tokens": 5183484.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.40999117493629456,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 51,
|
|
"step_time": 341.01204745844007
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3549.0,
|
|
"completions/mean_length": 1386.125,
|
|
"completions/mean_terminated_length": 999.0000610351562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.865479376167059,
|
|
"epoch": 0.02947845804988662,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.177734375,
|
|
"kl": 0.00048682725173421204,
|
|
"learning_rate": 9.710884353741497e-07,
|
|
"loss": 0.0312,
|
|
"num_tokens": 5282884.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 52,
|
|
"step_time": 293.0214045289904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.234375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3679.0,
|
|
"completions/mean_length": 1611.1875,
|
|
"completions/mean_terminated_length": 850.5305786132812,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.6968845427036285,
|
|
"epoch": 0.030045351473922902,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1611328125,
|
|
"kl": 0.00048145676919375546,
|
|
"learning_rate": 9.705215419501134e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 5397376.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 53,
|
|
"step_time": 295.37289924081415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3980.0,
|
|
"completions/mean_length": 1000.234375,
|
|
"completions/mean_terminated_length": 847.9835815429688,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.3745584413409233,
|
|
"epoch": 0.030612244897959183,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1259765625,
|
|
"kl": 0.000502369279274717,
|
|
"learning_rate": 9.69954648526077e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 5473719.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.613990306854248,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 54,
|
|
"step_time": 296.9595435075462
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3687.0,
|
|
"completions/mean_length": 1366.578125,
|
|
"completions/mean_terminated_length": 919.9454345703125,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 4.502534218132496,
|
|
"epoch": 0.031179138321995464,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.00041477979357296135,
|
|
"learning_rate": 9.693877551020407e-07,
|
|
"loss": 0.0287,
|
|
"num_tokens": 5573684.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 55,
|
|
"step_time": 296.263409120962
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3797.0,
|
|
"completions/mean_length": 1568.328125,
|
|
"completions/mean_terminated_length": 1100.24072265625,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 3.88920546323061,
|
|
"epoch": 0.031746031746031744,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.12109375,
|
|
"kl": 0.0004977654971298762,
|
|
"learning_rate": 9.688208616780045e-07,
|
|
"loss": 0.0257,
|
|
"num_tokens": 5687057.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 56,
|
|
"step_time": 297.1691315295175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3718.0,
|
|
"completions/mean_length": 1286.59375,
|
|
"completions/mean_terminated_length": 941.5789794921875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.255432682111859,
|
|
"epoch": 0.03231292517006803,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.16796875,
|
|
"kl": 0.0004884136069449596,
|
|
"learning_rate": 9.682539682539682e-07,
|
|
"loss": 0.0592,
|
|
"num_tokens": 5783583.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 57,
|
|
"step_time": 309.3456033077091
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3141.0,
|
|
"completions/mean_length": 871.546875,
|
|
"completions/mean_terminated_length": 598.2881469726562,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.336066670715809,
|
|
"epoch": 0.032879818594104306,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.427734375,
|
|
"kl": 0.0005954704392934218,
|
|
"learning_rate": 9.67687074829932e-07,
|
|
"loss": 0.0393,
|
|
"num_tokens": 5849834.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 58,
|
|
"step_time": 292.01666195411235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3611.0,
|
|
"completions/mean_length": 983.6875,
|
|
"completions/mean_terminated_length": 776.2000122070312,
|
|
"completions/min_length": 26.0,
|
|
"completions/min_terminated_length": 26.0,
|
|
"entropy": 2.50449425727129,
|
|
"epoch": 0.03344671201814059,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.0006079958839109167,
|
|
"learning_rate": 9.671201814058957e-07,
|
|
"loss": 0.0248,
|
|
"num_tokens": 5924790.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 59,
|
|
"step_time": 295.90404415596277
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3673.0,
|
|
"completions/mean_length": 1256.65625,
|
|
"completions/mean_terminated_length": 792.0363159179688,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 3.7395902425050735,
|
|
"epoch": 0.034013605442176874,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00010395050048828125,
|
|
"kl": 0.0005482348942678072,
|
|
"learning_rate": 9.665532879818595e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 6017456.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 60,
|
|
"step_time": 299.53357178531587
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4025.0,
|
|
"completions/mean_length": 1479.203125,
|
|
"completions/mean_terminated_length": 1105.375,
|
|
"completions/min_length": 263.0,
|
|
"completions/min_terminated_length": 263.0,
|
|
"entropy": 3.7954771779477596,
|
|
"epoch": 0.03458049886621315,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.0004666336762966239,
|
|
"learning_rate": 9.65986394557823e-07,
|
|
"loss": 0.0083,
|
|
"num_tokens": 6124485.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 61,
|
|
"step_time": 303.2139684362337
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4038.0,
|
|
"completions/mean_length": 1400.875,
|
|
"completions/mean_terminated_length": 959.8544921875,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.448891811072826,
|
|
"epoch": 0.035147392290249435,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.138671875,
|
|
"kl": 0.0005504604605448549,
|
|
"learning_rate": 9.654195011337868e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 6224125.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 62,
|
|
"step_time": 294.8489862466231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4077.0,
|
|
"completions/mean_length": 1326.484375,
|
|
"completions/mean_terminated_length": 930.83935546875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.4797469154000282,
|
|
"epoch": 0.03571428571428571,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2236328125,
|
|
"kl": 0.0005405677475209814,
|
|
"learning_rate": 9.648526077097505e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 6320292.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 63,
|
|
"step_time": 294.98920669872314
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3665.0,
|
|
"completions/mean_length": 1381.421875,
|
|
"completions/mean_terminated_length": 1048.0526123046875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.080914679914713,
|
|
"epoch": 0.036281179138321996,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.28125,
|
|
"kl": 0.0004902618484265986,
|
|
"learning_rate": 9.642857142857142e-07,
|
|
"loss": -0.0307,
|
|
"num_tokens": 6422399.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 64,
|
|
"step_time": 300.90544505324215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3904.0,
|
|
"completions/mean_length": 1388.59375,
|
|
"completions/mean_terminated_length": 887.2222290039062,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.027115806937218,
|
|
"epoch": 0.036848072562358274,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2255859375,
|
|
"kl": 0.0006420780437110807,
|
|
"learning_rate": 9.63718820861678e-07,
|
|
"loss": 0.012,
|
|
"num_tokens": 6523989.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 65,
|
|
"step_time": 298.16601424105465
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4046.0,
|
|
"completions/mean_length": 1429.3125,
|
|
"completions/mean_terminated_length": 992.9454345703125,
|
|
"completions/min_length": 158.0,
|
|
"completions/min_terminated_length": 158.0,
|
|
"entropy": 3.2920237369835377,
|
|
"epoch": 0.03741496598639456,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.142578125,
|
|
"kl": 0.0005064156493972405,
|
|
"learning_rate": 9.631519274376417e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 6629153.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 66,
|
|
"step_time": 300.80143326800317
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3894.0,
|
|
"completions/mean_length": 1645.015625,
|
|
"completions/mean_terminated_length": 958.739990234375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.213624369353056,
|
|
"epoch": 0.03798185941043084,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.2158203125,
|
|
"kl": 0.0004907570573777775,
|
|
"learning_rate": 9.625850340136053e-07,
|
|
"loss": 0.0149,
|
|
"num_tokens": 6746194.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 67,
|
|
"step_time": 295.89689590316266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3982.0,
|
|
"completions/mean_length": 1378.15625,
|
|
"completions/mean_terminated_length": 1044.385986328125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.088643457740545,
|
|
"epoch": 0.03854875283446712,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1591796875,
|
|
"kl": 0.00045124743974156445,
|
|
"learning_rate": 9.62018140589569e-07,
|
|
"loss": -0.0338,
|
|
"num_tokens": 6845908.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 68,
|
|
"step_time": 296.33633978664875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3903.0,
|
|
"completions/mean_length": 1253.875,
|
|
"completions/mean_terminated_length": 847.857177734375,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 2.7947971746325493,
|
|
"epoch": 0.0391156462585034,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.0005858802542206831,
|
|
"learning_rate": 9.614512471655328e-07,
|
|
"loss": 0.0518,
|
|
"num_tokens": 6939060.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 69,
|
|
"step_time": 297.3353471541777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3946.0,
|
|
"completions/mean_length": 1359.484375,
|
|
"completions/mean_terminated_length": 852.7222290039062,
|
|
"completions/min_length": 23.0,
|
|
"completions/min_terminated_length": 23.0,
|
|
"entropy": 3.448988128453493,
|
|
"epoch": 0.03968253968253968,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1123046875,
|
|
"kl": 0.0007160816503528622,
|
|
"learning_rate": 9.608843537414965e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 7037675.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.32732683420181274,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 70,
|
|
"step_time": 293.6382048111409
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3997.0,
|
|
"completions/mean_length": 1022.046875,
|
|
"completions/mean_terminated_length": 817.11669921875,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 3.8905701339244843,
|
|
"epoch": 0.040249433106575964,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.287109375,
|
|
"kl": 0.0005872798137716018,
|
|
"learning_rate": 9.603174603174603e-07,
|
|
"loss": 0.0793,
|
|
"num_tokens": 7114054.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 71,
|
|
"step_time": 293.8014563927427
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2784.0,
|
|
"completions/mean_length": 938.9375,
|
|
"completions/mean_terminated_length": 487.9285888671875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 1.9663578495383263,
|
|
"epoch": 0.04081632653061224,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.259765625,
|
|
"kl": 0.0007659744123884593,
|
|
"learning_rate": 9.59750566893424e-07,
|
|
"loss": 0.019,
|
|
"num_tokens": 7183450.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 72,
|
|
"step_time": 289.2986155534163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4064.0,
|
|
"completions/mean_length": 1038.734375,
|
|
"completions/mean_terminated_length": 722.4655151367188,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.3653258718550205,
|
|
"epoch": 0.041383219954648526,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.306640625,
|
|
"kl": 0.0006311132347036619,
|
|
"learning_rate": 9.591836734693876e-07,
|
|
"loss": 0.0313,
|
|
"num_tokens": 7269185.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 73,
|
|
"step_time": 375.2334147216752
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3538.0,
|
|
"completions/mean_length": 881.609375,
|
|
"completions/mean_terminated_length": 667.3167114257812,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 2.874113205820322,
|
|
"epoch": 0.04195011337868481,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2412109375,
|
|
"kl": 0.0006657410795014584,
|
|
"learning_rate": 9.586167800453513e-07,
|
|
"loss": 0.0335,
|
|
"num_tokens": 7336264.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 74,
|
|
"step_time": 290.8858574004844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3541.0,
|
|
"completions/mean_length": 1720.453125,
|
|
"completions/mean_terminated_length": 1114.921630859375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.012122828513384,
|
|
"epoch": 0.04251700680272109,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.0004141690405958798,
|
|
"learning_rate": 9.58049886621315e-07,
|
|
"loss": 0.0161,
|
|
"num_tokens": 7457877.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 75,
|
|
"step_time": 296.16961095202714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3963.0,
|
|
"completions/mean_length": 1003.28125,
|
|
"completions/mean_terminated_length": 683.3448486328125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.2306234762072563,
|
|
"epoch": 0.04308390022675737,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.000566702869036817,
|
|
"learning_rate": 9.574829931972788e-07,
|
|
"loss": 0.0374,
|
|
"num_tokens": 7532191.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7303985953330994,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 76,
|
|
"step_time": 290.138320222497
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3892.0,
|
|
"completions/mean_length": 1390.09375,
|
|
"completions/mean_terminated_length": 1003.5357666015625,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 3.7910470850765705,
|
|
"epoch": 0.04365079365079365,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.173828125,
|
|
"kl": 0.0005802920577480108,
|
|
"learning_rate": 9.569160997732426e-07,
|
|
"loss": -0.0138,
|
|
"num_tokens": 7632333.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 77,
|
|
"step_time": 293.7238145461306
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3881.0,
|
|
"completions/mean_length": 1424.296875,
|
|
"completions/mean_terminated_length": 1096.1929931640625,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.150249805301428,
|
|
"epoch": 0.04421768707482993,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.236328125,
|
|
"kl": 0.0005237193527136697,
|
|
"learning_rate": 9.563492063492063e-07,
|
|
"loss": 0.0141,
|
|
"num_tokens": 7736216.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 78,
|
|
"step_time": 299.65833119209856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4074.0,
|
|
"completions/mean_length": 1354.09375,
|
|
"completions/mean_terminated_length": 962.3928833007812,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 4.0368705578148365,
|
|
"epoch": 0.04478458049886622,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.0005636081277771154,
|
|
"learning_rate": 9.5578231292517e-07,
|
|
"loss": 0.0239,
|
|
"num_tokens": 7834734.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 79,
|
|
"step_time": 296.3580552712083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3816.0,
|
|
"completions/mean_length": 1199.765625,
|
|
"completions/mean_terminated_length": 844.0877075195312,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.758032266050577,
|
|
"epoch": 0.045351473922902494,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2392578125,
|
|
"kl": 0.000635372903161624,
|
|
"learning_rate": 9.552154195011338e-07,
|
|
"loss": 0.0212,
|
|
"num_tokens": 7923047.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 80,
|
|
"step_time": 294.42624179366976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3742.0,
|
|
"completions/mean_length": 1513.125,
|
|
"completions/mean_terminated_length": 1090.47265625,
|
|
"completions/min_length": 268.0,
|
|
"completions/min_terminated_length": 268.0,
|
|
"entropy": 3.8117652274668217,
|
|
"epoch": 0.04591836734693878,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00011539459228515625,
|
|
"kl": 0.0005118364797453978,
|
|
"learning_rate": 9.546485260770974e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 8030687.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 81,
|
|
"step_time": 291.95394702255726
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3812.0,
|
|
"completions/mean_length": 1353.421875,
|
|
"completions/mean_terminated_length": 784.2075805664062,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.888309068977833,
|
|
"epoch": 0.046485260770975055,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.13671875,
|
|
"kl": 0.0005780409746876103,
|
|
"learning_rate": 9.54081632653061e-07,
|
|
"loss": 0.0015,
|
|
"num_tokens": 8133042.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 82,
|
|
"step_time": 329.26528967916965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4070.0,
|
|
"completions/mean_length": 1369.96875,
|
|
"completions/mean_terminated_length": 1035.1929931640625,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.8123384825885296,
|
|
"epoch": 0.04705215419501134,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.00047865481610642746,
|
|
"learning_rate": 9.53514739229025e-07,
|
|
"loss": -0.1455,
|
|
"num_tokens": 8231048.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 83,
|
|
"step_time": 293.00953847169876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3999.0,
|
|
"completions/mean_length": 1308.75,
|
|
"completions/mean_terminated_length": 852.654541015625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.287667583674192,
|
|
"epoch": 0.047619047619047616,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.21484375,
|
|
"kl": 0.000524261982718599,
|
|
"learning_rate": 9.529478458049887e-07,
|
|
"loss": 0.0314,
|
|
"num_tokens": 8326752.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 84,
|
|
"step_time": 294.7698496468365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3809.0,
|
|
"completions/mean_length": 1655.015625,
|
|
"completions/mean_terminated_length": 1091.7115478515625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.714879844337702,
|
|
"epoch": 0.0481859410430839,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2197265625,
|
|
"kl": 0.0005542064991459483,
|
|
"learning_rate": 9.523809523809522e-07,
|
|
"loss": 0.0609,
|
|
"num_tokens": 8445641.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 85,
|
|
"step_time": 295.54682763200253
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4049.0,
|
|
"completions/mean_length": 1364.390625,
|
|
"completions/mean_terminated_length": 974.1607666015625,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 3.585228767246008,
|
|
"epoch": 0.048752834467120185,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1513671875,
|
|
"kl": 0.0005338712680895696,
|
|
"learning_rate": 9.51814058956916e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 8545090.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 86,
|
|
"step_time": 297.61800406873226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3979.0,
|
|
"completions/mean_length": 1530.359375,
|
|
"completions/mean_terminated_length": 1055.24072265625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.526728063821793,
|
|
"epoch": 0.04931972789115646,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.0005592237139353529,
|
|
"learning_rate": 9.512471655328797e-07,
|
|
"loss": 0.0105,
|
|
"num_tokens": 8657049.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 87,
|
|
"step_time": 302.05309406947345
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3760.0,
|
|
"completions/mean_length": 1434.265625,
|
|
"completions/mean_terminated_length": 1054.0179443359375,
|
|
"completions/min_length": 186.0,
|
|
"completions/min_terminated_length": 186.0,
|
|
"entropy": 4.485512547194958,
|
|
"epoch": 0.049886621315192746,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2197265625,
|
|
"kl": 0.000496680951982853,
|
|
"learning_rate": 9.506802721088435e-07,
|
|
"loss": 0.036,
|
|
"num_tokens": 8769642.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 88,
|
|
"step_time": 371.294574201107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3597.0,
|
|
"completions/mean_length": 1918.34375,
|
|
"completions/mean_terminated_length": 854.8372192382812,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 2.781709646806121,
|
|
"epoch": 0.05045351473922902,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1103515625,
|
|
"kl": 0.000628817981123575,
|
|
"learning_rate": 9.501133786848072e-07,
|
|
"loss": 0.0114,
|
|
"num_tokens": 8904432.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 89,
|
|
"step_time": 301.35453115589917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3035.0,
|
|
"completions/mean_length": 1303.71875,
|
|
"completions/mean_terminated_length": 846.7999877929688,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.7423397190868855,
|
|
"epoch": 0.05102040816326531,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.251953125,
|
|
"kl": 0.0005851384448760655,
|
|
"learning_rate": 9.49546485260771e-07,
|
|
"loss": 0.0313,
|
|
"num_tokens": 9000566.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 90,
|
|
"step_time": 295.8532868111506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3730.0,
|
|
"completions/mean_length": 1143.890625,
|
|
"completions/mean_terminated_length": 781.3508911132812,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.400463607162237,
|
|
"epoch": 0.051587301587301584,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.0006745926693838555,
|
|
"learning_rate": 9.489795918367347e-07,
|
|
"loss": 0.0016,
|
|
"num_tokens": 9087631.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 91,
|
|
"step_time": 310.4487501978874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3783.0,
|
|
"completions/mean_length": 1199.046875,
|
|
"completions/mean_terminated_length": 953.5423583984375,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 3.66338399797678,
|
|
"epoch": 0.05215419501133787,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.0005909999781579245,
|
|
"learning_rate": 9.484126984126984e-07,
|
|
"loss": 0.0785,
|
|
"num_tokens": 9175770.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 92,
|
|
"step_time": 296.0928402291611
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4031.0,
|
|
"completions/mean_length": 1609.140625,
|
|
"completions/mean_terminated_length": 1148.611083984375,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 5.085915923118591,
|
|
"epoch": 0.05272108843537415,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1484375,
|
|
"kl": 0.00045808015329384943,
|
|
"learning_rate": 9.47845804988662e-07,
|
|
"loss": 0.0198,
|
|
"num_tokens": 9291971.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 93,
|
|
"step_time": 305.8303446471691
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3274.0,
|
|
"completions/mean_length": 1377.671875,
|
|
"completions/mean_terminated_length": 750.3654174804688,
|
|
"completions/min_length": 46.0,
|
|
"completions/min_terminated_length": 46.0,
|
|
"entropy": 3.596812978386879,
|
|
"epoch": 0.05328798185941043,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2294921875,
|
|
"kl": 0.0006529647143906914,
|
|
"learning_rate": 9.472789115646258e-07,
|
|
"loss": -0.0227,
|
|
"num_tokens": 9392806.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 94,
|
|
"step_time": 298.4613697146997
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2999.0,
|
|
"completions/mean_length": 987.5,
|
|
"completions/mean_terminated_length": 780.2667236328125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.4541202299296856,
|
|
"epoch": 0.053854875283446714,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.0006040437892806949,
|
|
"learning_rate": 9.467120181405895e-07,
|
|
"loss": -0.0633,
|
|
"num_tokens": 9467174.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 95,
|
|
"step_time": 294.7388898720965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4009.0,
|
|
"completions/mean_length": 1225.59375,
|
|
"completions/mean_terminated_length": 928.6551513671875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.783916585147381,
|
|
"epoch": 0.05442176870748299,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.000592722135479562,
|
|
"learning_rate": 9.461451247165533e-07,
|
|
"loss": 0.0735,
|
|
"num_tokens": 9557764.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 96,
|
|
"step_time": 302.38117443956435
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4062.0,
|
|
"completions/mean_length": 1765.25,
|
|
"completions/mean_terminated_length": 1227.3846435546875,
|
|
"completions/min_length": 189.0,
|
|
"completions/min_terminated_length": 189.0,
|
|
"entropy": 4.494186479598284,
|
|
"epoch": 0.054988662131519275,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.10888671875,
|
|
"kl": 0.00047747676944709383,
|
|
"learning_rate": 9.45578231292517e-07,
|
|
"loss": 0.0183,
|
|
"num_tokens": 9683260.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.3125,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 97,
|
|
"step_time": 299.90580482501537
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3940.0,
|
|
"completions/mean_length": 1477.703125,
|
|
"completions/mean_terminated_length": 1049.2545166015625,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 3.7976103238761425,
|
|
"epoch": 0.05555555555555555,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.244140625,
|
|
"kl": 0.0005473416022141464,
|
|
"learning_rate": 9.450113378684807e-07,
|
|
"loss": -0.0045,
|
|
"num_tokens": 9790001.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 98,
|
|
"step_time": 294.92261796910316
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3028.0,
|
|
"completions/mean_length": 1345.3125,
|
|
"completions/mean_terminated_length": 835.9259033203125,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.06356118991971,
|
|
"epoch": 0.05612244897959184,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2099609375,
|
|
"kl": 0.0006883190781081794,
|
|
"learning_rate": 9.444444444444444e-07,
|
|
"loss": 0.0357,
|
|
"num_tokens": 9890381.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 99,
|
|
"step_time": 321.9701617071405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4009.0,
|
|
"completions/mean_length": 1932.703125,
|
|
"completions/mean_terminated_length": 1211.604248046875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.165743313729763,
|
|
"epoch": 0.05668934240362812,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.15234375,
|
|
"kl": 0.0005034771747887135,
|
|
"learning_rate": 9.438775510204082e-07,
|
|
"loss": 0.0204,
|
|
"num_tokens": 10026154.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.4005824029445648,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 100,
|
|
"step_time": 297.60639240499586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3944.0,
|
|
"completions/mean_length": 1256.65625,
|
|
"completions/mean_terminated_length": 792.0363159179688,
|
|
"completions/min_length": 29.0,
|
|
"completions/min_terminated_length": 29.0,
|
|
"entropy": 3.866720013320446,
|
|
"epoch": 0.0572562358276644,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.15234375,
|
|
"kl": 0.0006163184216347872,
|
|
"learning_rate": 9.433106575963718e-07,
|
|
"loss": 0.0163,
|
|
"num_tokens": 10117540.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 101,
|
|
"step_time": 294.1511483108625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4069.0,
|
|
"completions/mean_length": 938.515625,
|
|
"completions/mean_terminated_length": 836.6612548828125,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.3969429675489664,
|
|
"epoch": 0.05782312925170068,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1416015625,
|
|
"kl": 0.0005263586681394372,
|
|
"learning_rate": 9.427437641723356e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 10188845.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3398725986480713,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 102,
|
|
"step_time": 293.2026412822306
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3905.0,
|
|
"completions/mean_length": 1319.109375,
|
|
"completions/mean_terminated_length": 804.870361328125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.2274572625756264,
|
|
"epoch": 0.05839002267573696,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1220703125,
|
|
"kl": 0.0006206169309734832,
|
|
"learning_rate": 9.421768707482993e-07,
|
|
"loss": -0.0151,
|
|
"num_tokens": 10284436.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.32265254855155945,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 103,
|
|
"step_time": 294.4274990260601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4060.0,
|
|
"completions/mean_length": 1213.171875,
|
|
"completions/mean_terminated_length": 914.9482421875,
|
|
"completions/min_length": 37.0,
|
|
"completions/min_terminated_length": 37.0,
|
|
"entropy": 3.7609485015273094,
|
|
"epoch": 0.05895691609977324,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2119140625,
|
|
"kl": 0.0006420473982871044,
|
|
"learning_rate": 9.41609977324263e-07,
|
|
"loss": 0.0083,
|
|
"num_tokens": 10375711.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 104,
|
|
"step_time": 301.6561495643109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3825.0,
|
|
"completions/mean_length": 1011.78125,
|
|
"completions/mean_terminated_length": 692.72412109375,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 2.2102625109255314,
|
|
"epoch": 0.05952380952380952,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.35546875,
|
|
"kl": 0.0006728825683239847,
|
|
"learning_rate": 9.410430839002267e-07,
|
|
"loss": 0.0437,
|
|
"num_tokens": 10451249.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 105,
|
|
"step_time": 293.18639434315264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3894.0,
|
|
"completions/mean_length": 1096.640625,
|
|
"completions/mean_terminated_length": 842.4576416015625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.8951598592102528,
|
|
"epoch": 0.060090702947845805,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.23828125,
|
|
"kl": 0.0006520739998450154,
|
|
"learning_rate": 9.404761904761904e-07,
|
|
"loss": 0.0211,
|
|
"num_tokens": 10532410.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 106,
|
|
"step_time": 295.260443424806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3860.0,
|
|
"completions/mean_length": 1784.734375,
|
|
"completions/mean_terminated_length": 1195.5882568359375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 4.265113152563572,
|
|
"epoch": 0.06065759637188209,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1552734375,
|
|
"kl": 0.00046251650746853556,
|
|
"learning_rate": 9.399092970521542e-07,
|
|
"loss": -0.0044,
|
|
"num_tokens": 10659921.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 107,
|
|
"step_time": 303.3586766477674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3901.0,
|
|
"completions/mean_length": 1430.84375,
|
|
"completions/mean_terminated_length": 1050.107177734375,
|
|
"completions/min_length": 154.0,
|
|
"completions/min_terminated_length": 154.0,
|
|
"entropy": 3.967142190784216,
|
|
"epoch": 0.061224489795918366,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1357421875,
|
|
"kl": 0.0005185274985706201,
|
|
"learning_rate": 9.393424036281179e-07,
|
|
"loss": 0.0245,
|
|
"num_tokens": 10762111.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 108,
|
|
"step_time": 294.1047155074775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3891.0,
|
|
"completions/mean_length": 1249.171875,
|
|
"completions/mean_terminated_length": 899.5614013671875,
|
|
"completions/min_length": 23.0,
|
|
"completions/min_terminated_length": 23.0,
|
|
"entropy": 3.897375989705324,
|
|
"epoch": 0.06179138321995465,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2060546875,
|
|
"kl": 0.0006039557038093335,
|
|
"learning_rate": 9.387755102040816e-07,
|
|
"loss": 0.0343,
|
|
"num_tokens": 10852498.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5820031762123108,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 109,
|
|
"step_time": 294.5808366378769
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3963.0,
|
|
"completions/mean_length": 1259.546875,
|
|
"completions/mean_terminated_length": 795.3999633789062,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.542051460593939,
|
|
"epoch": 0.06235827664399093,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2353515625,
|
|
"kl": 0.0007039975898806006,
|
|
"learning_rate": 9.382086167800452e-07,
|
|
"loss": 0.0688,
|
|
"num_tokens": 10944333.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 110,
|
|
"step_time": 293.68635532446206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4064.0,
|
|
"completions/mean_length": 1230.65625,
|
|
"completions/mean_terminated_length": 878.7719116210938,
|
|
"completions/min_length": 21.0,
|
|
"completions/min_terminated_length": 21.0,
|
|
"entropy": 3.576985750347376,
|
|
"epoch": 0.06292517006802721,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1572265625,
|
|
"kl": 0.0005771926189481746,
|
|
"learning_rate": 9.37641723356009e-07,
|
|
"loss": 0.0235,
|
|
"num_tokens": 11036399.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 111,
|
|
"step_time": 306.3100119102746
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3691.0,
|
|
"completions/mean_length": 1242.515625,
|
|
"completions/mean_terminated_length": 650.2830200195312,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 2.644986890256405,
|
|
"epoch": 0.06349206349206349,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.140625,
|
|
"kl": 0.0007677831636101473,
|
|
"learning_rate": 9.370748299319727e-07,
|
|
"loss": 0.0263,
|
|
"num_tokens": 11127544.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.5379911661148071,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 112,
|
|
"step_time": 296.9050910929218
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4023.0,
|
|
"completions/mean_length": 1472.0,
|
|
"completions/mean_terminated_length": 986.0740966796875,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"entropy": 4.75571757927537,
|
|
"epoch": 0.06405895691609978,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.10009765625,
|
|
"kl": 0.0005265878962745774,
|
|
"learning_rate": 9.365079365079365e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 11234496.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5395448207855225,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 113,
|
|
"step_time": 312.21911957859993
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3857.0,
|
|
"completions/mean_length": 1244.5,
|
|
"completions/mean_terminated_length": 716.4444580078125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.6412566378712654,
|
|
"epoch": 0.06462585034013606,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.150390625,
|
|
"kl": 0.0007886644025347778,
|
|
"learning_rate": 9.359410430839002e-07,
|
|
"loss": -0.0186,
|
|
"num_tokens": 11327856.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 114,
|
|
"step_time": 313.32709429971874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3941.0,
|
|
"completions/mean_length": 1519.6875,
|
|
"completions/mean_terminated_length": 798.3200073242188,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.032174177467823,
|
|
"epoch": 0.06519274376417233,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.263671875,
|
|
"kl": 0.0008032457553781569,
|
|
"learning_rate": 9.35374149659864e-07,
|
|
"loss": 0.0317,
|
|
"num_tokens": 11436668.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 115,
|
|
"step_time": 296.70230418629944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3928.0,
|
|
"completions/mean_length": 818.59375,
|
|
"completions/mean_terminated_length": 657.4097900390625,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 2.9340442791581154,
|
|
"epoch": 0.06575963718820861,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.000689146967488341,
|
|
"learning_rate": 9.348072562358276e-07,
|
|
"loss": 0.0312,
|
|
"num_tokens": 11506282.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 116,
|
|
"step_time": 338.6807353729382
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3776.0,
|
|
"completions/mean_length": 1385.96875,
|
|
"completions/mean_terminated_length": 998.8214721679688,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.467217389494181,
|
|
"epoch": 0.0663265306122449,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.166015625,
|
|
"kl": 0.00047883675961202243,
|
|
"learning_rate": 9.342403628117913e-07,
|
|
"loss": 0.0138,
|
|
"num_tokens": 11609216.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 117,
|
|
"step_time": 313.09567215200514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3807.0,
|
|
"completions/mean_length": 826.984375,
|
|
"completions/mean_terminated_length": 609.050048828125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.981639474630356,
|
|
"epoch": 0.06689342403628118,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.20703125,
|
|
"kl": 0.0006725377279508393,
|
|
"learning_rate": 9.33673469387755e-07,
|
|
"loss": -0.0067,
|
|
"num_tokens": 11675175.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.653468668460846,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 118,
|
|
"step_time": 309.6762728523463
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3890.0,
|
|
"completions/mean_length": 1198.265625,
|
|
"completions/mean_terminated_length": 842.4035034179688,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.0181843265891075,
|
|
"epoch": 0.06746031746031746,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2177734375,
|
|
"kl": 0.0006909497224114602,
|
|
"learning_rate": 9.331065759637188e-07,
|
|
"loss": 0.0041,
|
|
"num_tokens": 11761832.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 119,
|
|
"step_time": 291.38728551380336
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.234375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4093.0,
|
|
"completions/mean_length": 1715.453125,
|
|
"completions/mean_terminated_length": 986.7142944335938,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.1485245525836945,
|
|
"epoch": 0.06802721088435375,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1513671875,
|
|
"kl": 0.0005977986420475645,
|
|
"learning_rate": 9.325396825396825e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 11883405.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 120,
|
|
"step_time": 296.01865003444254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3722.0,
|
|
"completions/mean_length": 1502.234375,
|
|
"completions/mean_terminated_length": 841.0784912109375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.063643492758274,
|
|
"epoch": 0.06859410430839002,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1640625,
|
|
"kl": 0.0007376213197858306,
|
|
"learning_rate": 9.319727891156463e-07,
|
|
"loss": 0.0133,
|
|
"num_tokens": 11992156.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 121,
|
|
"step_time": 302.0782301174477
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3696.0,
|
|
"completions/mean_length": 1317.6875,
|
|
"completions/mean_terminated_length": 863.0545043945312,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.4850979894399643,
|
|
"epoch": 0.0691609977324263,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.244140625,
|
|
"kl": 0.0006848964658274781,
|
|
"learning_rate": 9.3140589569161e-07,
|
|
"loss": 0.0451,
|
|
"num_tokens": 12088696.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 122,
|
|
"step_time": 300.00162667408586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3851.0,
|
|
"completions/mean_length": 1209.234375,
|
|
"completions/mean_terminated_length": 796.8392944335938,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 4.3270882070064545,
|
|
"epoch": 0.06972789115646258,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.263671875,
|
|
"kl": 0.0006902841369083035,
|
|
"learning_rate": 9.308390022675737e-07,
|
|
"loss": 0.0542,
|
|
"num_tokens": 12180911.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 123,
|
|
"step_time": 306.8877000166103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4032.0,
|
|
"completions/mean_length": 1511.90625,
|
|
"completions/mean_terminated_length": 1033.370361328125,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 5.586065195500851,
|
|
"epoch": 0.07029478458049887,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1689453125,
|
|
"kl": 0.0005649285230902024,
|
|
"learning_rate": 9.302721088435374e-07,
|
|
"loss": 0.061,
|
|
"num_tokens": 12288641.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 124,
|
|
"step_time": 296.0335800014436
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4025.0,
|
|
"completions/mean_length": 1398.34375,
|
|
"completions/mean_terminated_length": 1012.96435546875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.474613543599844,
|
|
"epoch": 0.07086167800453515,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00015354156494140625,
|
|
"kl": 0.0006384559055732097,
|
|
"learning_rate": 9.29705215419501e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 12391743.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.1666666716337204,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 125,
|
|
"step_time": 300.94666452147067
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4047.0,
|
|
"completions/mean_length": 1286.171875,
|
|
"completions/mean_terminated_length": 884.7678833007812,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.363495498895645,
|
|
"epoch": 0.07142857142857142,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.296875,
|
|
"kl": 0.0007903738214736222,
|
|
"learning_rate": 9.291383219954648e-07,
|
|
"loss": -0.0225,
|
|
"num_tokens": 12486010.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 126,
|
|
"step_time": 298.92847953084856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3481.0,
|
|
"completions/mean_length": 1562.15625,
|
|
"completions/mean_terminated_length": 977.423095703125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.3883329667150974,
|
|
"epoch": 0.07199546485260772,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.173828125,
|
|
"kl": 0.0006784117667848477,
|
|
"learning_rate": 9.285714285714285e-07,
|
|
"loss": 0.0166,
|
|
"num_tokens": 12598372.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 127,
|
|
"step_time": 296.65200543869287
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2759.0,
|
|
"completions/mean_length": 989.515625,
|
|
"completions/mean_terminated_length": 668.1551513671875,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.646180473268032,
|
|
"epoch": 0.07256235827664399,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.0008213972214434762,
|
|
"learning_rate": 9.280045351473923e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 12673509.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 128,
|
|
"step_time": 295.8863944094628
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4078.0,
|
|
"completions/mean_length": 1332.421875,
|
|
"completions/mean_terminated_length": 820.6481323242188,
|
|
"completions/min_length": 83.0,
|
|
"completions/min_terminated_length": 83.0,
|
|
"entropy": 2.9267875254154205,
|
|
"epoch": 0.07312925170068027,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2099609375,
|
|
"kl": 0.000708395169567666,
|
|
"learning_rate": 9.274376417233559e-07,
|
|
"loss": 0.0156,
|
|
"num_tokens": 12769648.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 129,
|
|
"step_time": 294.252164516598
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3993.0,
|
|
"completions/mean_length": 1347.3125,
|
|
"completions/mean_terminated_length": 1062.9654541015625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.332023099064827,
|
|
"epoch": 0.07369614512471655,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.000581516707825358,
|
|
"learning_rate": 9.268707482993197e-07,
|
|
"loss": 0.0189,
|
|
"num_tokens": 12867388.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 130,
|
|
"step_time": 296.34269182849675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.234375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2999.0,
|
|
"completions/mean_length": 1607.640625,
|
|
"completions/mean_terminated_length": 845.89794921875,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 4.000800345093012,
|
|
"epoch": 0.07426303854875284,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.166015625,
|
|
"kl": 0.0009954783781722654,
|
|
"learning_rate": 9.263038548752834e-07,
|
|
"loss": 0.0278,
|
|
"num_tokens": 12982205.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4917473793029785,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 131,
|
|
"step_time": 296.1071062963456
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3894.0,
|
|
"completions/mean_length": 1362.4375,
|
|
"completions/mean_terminated_length": 856.2222290039062,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.674266640096903,
|
|
"epoch": 0.07482993197278912,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.12451171875,
|
|
"kl": 0.0006412754437405965,
|
|
"learning_rate": 9.257369614512472e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 13080969.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 132,
|
|
"step_time": 303.2742968611419
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3891.0,
|
|
"completions/mean_length": 1077.09375,
|
|
"completions/mean_terminated_length": 764.7930908203125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.4341964684426785,
|
|
"epoch": 0.07539682539682539,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.0007655090921616647,
|
|
"learning_rate": 9.251700680272108e-07,
|
|
"loss": -0.0576,
|
|
"num_tokens": 13161559.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 133,
|
|
"step_time": 296.6053422614932
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4039.0,
|
|
"completions/mean_length": 1340.3125,
|
|
"completions/mean_terminated_length": 946.6428833007812,
|
|
"completions/min_length": 107.0,
|
|
"completions/min_terminated_length": 107.0,
|
|
"entropy": 3.7123965956270695,
|
|
"epoch": 0.07596371882086168,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2177734375,
|
|
"kl": 0.0006033002264302922,
|
|
"learning_rate": 9.246031746031746e-07,
|
|
"loss": 0.0028,
|
|
"num_tokens": 13259563.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 134,
|
|
"step_time": 297.93012297712266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3623.0,
|
|
"completions/mean_length": 1054.921875,
|
|
"completions/mean_terminated_length": 620.482177734375,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"entropy": 2.5744658894836903,
|
|
"epoch": 0.07653061224489796,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.287109375,
|
|
"kl": 0.0008069383547990583,
|
|
"learning_rate": 9.240362811791382e-07,
|
|
"loss": 0.0362,
|
|
"num_tokens": 13337550.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 135,
|
|
"step_time": 292.86695772688836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4013.0,
|
|
"completions/mean_length": 1441.484375,
|
|
"completions/mean_terminated_length": 1115.4912109375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.732803553342819,
|
|
"epoch": 0.07709750566893424,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.00047232820179488044,
|
|
"learning_rate": 9.23469387755102e-07,
|
|
"loss": -0.0178,
|
|
"num_tokens": 13440885.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 136,
|
|
"step_time": 293.1682682046667
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2807.0,
|
|
"completions/mean_length": 700.015625,
|
|
"completions/mean_terminated_length": 533.0,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 2.496282309293747,
|
|
"epoch": 0.07766439909297052,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.0009696580927993637,
|
|
"learning_rate": 9.229024943310657e-07,
|
|
"loss": -0.002,
|
|
"num_tokens": 13497606.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.7451896071434021,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 137,
|
|
"step_time": 294.9174865875393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3858.0,
|
|
"completions/mean_length": 1188.640625,
|
|
"completions/mean_terminated_length": 994.8167114257812,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.9258103258907795,
|
|
"epoch": 0.0782312925170068,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.0006540652311741724,
|
|
"learning_rate": 9.223356009070295e-07,
|
|
"loss": 0.0428,
|
|
"num_tokens": 13588127.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 138,
|
|
"step_time": 316.7908872719854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3090.0,
|
|
"completions/mean_length": 1407.359375,
|
|
"completions/mean_terminated_length": 967.3999633789062,
|
|
"completions/min_length": 127.0,
|
|
"completions/min_terminated_length": 127.0,
|
|
"entropy": 3.2234241291880608,
|
|
"epoch": 0.07879818594104308,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.0007198409548436757,
|
|
"learning_rate": 9.217687074829932e-07,
|
|
"loss": 0.0617,
|
|
"num_tokens": 13691222.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 139,
|
|
"step_time": 297.9994641896337
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3727.0,
|
|
"completions/mean_length": 1078.921875,
|
|
"completions/mean_terminated_length": 823.2373046875,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"entropy": 3.5499210990965366,
|
|
"epoch": 0.07936507936507936,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.00083128790538467,
|
|
"learning_rate": 9.21201814058957e-07,
|
|
"loss": 0.0825,
|
|
"num_tokens": 13770089.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 140,
|
|
"step_time": 290.72720505669713
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4028.0,
|
|
"completions/mean_length": 1039.96875,
|
|
"completions/mean_terminated_length": 723.8275756835938,
|
|
"completions/min_length": 160.0,
|
|
"completions/min_terminated_length": 160.0,
|
|
"entropy": 3.0028465539216995,
|
|
"epoch": 0.07993197278911565,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.34375,
|
|
"kl": 0.0008300166300614364,
|
|
"learning_rate": 9.206349206349205e-07,
|
|
"loss": -0.0147,
|
|
"num_tokens": 13847463.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5747131109237671,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 141,
|
|
"step_time": 292.8372485693544
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4023.0,
|
|
"completions/mean_length": 886.84375,
|
|
"completions/mean_terminated_length": 672.9000244140625,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 2.4044470489025116,
|
|
"epoch": 0.08049886621315193,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.0011324633942422224,
|
|
"learning_rate": 9.200680272108843e-07,
|
|
"loss": 0.0173,
|
|
"num_tokens": 13915485.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.33174845576286316,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 142,
|
|
"step_time": 294.4719723528251
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4092.0,
|
|
"completions/mean_length": 1612.421875,
|
|
"completions/mean_terminated_length": 1152.5,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"entropy": 4.717786908149719,
|
|
"epoch": 0.0810657596371882,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2158203125,
|
|
"kl": 0.0006212099324329756,
|
|
"learning_rate": 9.19501133786848e-07,
|
|
"loss": -0.0116,
|
|
"num_tokens": 14030184.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 143,
|
|
"step_time": 296.9507727250457
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3716.0,
|
|
"completions/mean_length": 1006.296875,
|
|
"completions/mean_terminated_length": 744.4576416015625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.8041425682604313,
|
|
"epoch": 0.08163265306122448,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.236328125,
|
|
"kl": 0.0008463433550787158,
|
|
"learning_rate": 9.189342403628118e-07,
|
|
"loss": 0.0045,
|
|
"num_tokens": 14106299.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 144,
|
|
"step_time": 294.0263333497569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3609.0,
|
|
"completions/mean_length": 1302.328125,
|
|
"completions/mean_terminated_length": 722.5094604492188,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.2059508860111237,
|
|
"epoch": 0.08219954648526077,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.18359375,
|
|
"kl": 0.0009646604430599837,
|
|
"learning_rate": 9.183673469387755e-07,
|
|
"loss": -0.0243,
|
|
"num_tokens": 14202584.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 145,
|
|
"step_time": 301.0547637110576
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3827.0,
|
|
"completions/mean_length": 938.734375,
|
|
"completions/mean_terminated_length": 728.2500610351562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.7331823147833347,
|
|
"epoch": 0.08276643990929705,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.000867834141899948,
|
|
"learning_rate": 9.178004535147393e-07,
|
|
"loss": 0.0108,
|
|
"num_tokens": 14277135.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 146,
|
|
"step_time": 311.6211075047031
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3997.0,
|
|
"completions/mean_length": 1596.484375,
|
|
"completions/mean_terminated_length": 1077.717041015625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.653958112001419,
|
|
"epoch": 0.08333333333333333,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1552734375,
|
|
"kl": 0.00059996831441822,
|
|
"learning_rate": 9.172335600907029e-07,
|
|
"loss": -0.0023,
|
|
"num_tokens": 14390822.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 147,
|
|
"step_time": 294.62713649868965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3383.0,
|
|
"completions/mean_length": 1330.984375,
|
|
"completions/mean_terminated_length": 935.982177734375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.549279760569334,
|
|
"epoch": 0.08390022675736962,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.236328125,
|
|
"kl": 0.0007655783811060246,
|
|
"learning_rate": 9.166666666666665e-07,
|
|
"loss": 0.0043,
|
|
"num_tokens": 14487165.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 148,
|
|
"step_time": 292.3098398465663
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3942.0,
|
|
"completions/mean_length": 1446.515625,
|
|
"completions/mean_terminated_length": 896.6226806640625,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 3.011128805577755,
|
|
"epoch": 0.0844671201814059,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.0007254601096065016,
|
|
"learning_rate": 9.160997732426303e-07,
|
|
"loss": 0.0292,
|
|
"num_tokens": 14591454.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 149,
|
|
"step_time": 295.3695561401546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3775.0,
|
|
"completions/mean_length": 1427.84375,
|
|
"completions/mean_terminated_length": 991.236328125,
|
|
"completions/min_length": 64.0,
|
|
"completions/min_terminated_length": 64.0,
|
|
"entropy": 3.2528973296284676,
|
|
"epoch": 0.08503401360544217,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1787109375,
|
|
"kl": 0.0006524867385451216,
|
|
"learning_rate": 9.15532879818594e-07,
|
|
"loss": 0.0224,
|
|
"num_tokens": 14694284.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 150,
|
|
"step_time": 295.72957531083375
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3824.0,
|
|
"completions/mean_length": 1812.015625,
|
|
"completions/mean_terminated_length": 1229.8236083984375,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 4.662757508456707,
|
|
"epoch": 0.08560090702947845,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.107421875,
|
|
"kl": 0.0005463318366309977,
|
|
"learning_rate": 9.149659863945578e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 14821861.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 151,
|
|
"step_time": 295.2445678981021
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3852.0,
|
|
"completions/mean_length": 1448.96875,
|
|
"completions/mean_terminated_length": 707.7999877929688,
|
|
"completions/min_length": 44.0,
|
|
"completions/min_terminated_length": 44.0,
|
|
"entropy": 2.8441838547587395,
|
|
"epoch": 0.08616780045351474,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1318359375,
|
|
"kl": 0.0010018190896516899,
|
|
"learning_rate": 9.143990929705215e-07,
|
|
"loss": -0.0178,
|
|
"num_tokens": 14926715.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 152,
|
|
"step_time": 296.1488225897774
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4022.0,
|
|
"completions/mean_length": 1175.703125,
|
|
"completions/mean_terminated_length": 1032.0819091796875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.440500654280186,
|
|
"epoch": 0.08673469387755102,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.0006847782442491734,
|
|
"learning_rate": 9.138321995464853e-07,
|
|
"loss": 0.0343,
|
|
"num_tokens": 15013960.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 153,
|
|
"step_time": 294.9221243271604
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4052.0,
|
|
"completions/mean_length": 1557.359375,
|
|
"completions/mean_terminated_length": 1087.24072265625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.584035370498896,
|
|
"epoch": 0.0873015873015873,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1259765625,
|
|
"kl": 0.0006917003101989394,
|
|
"learning_rate": 9.132653061224489e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 15124903.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 154,
|
|
"step_time": 296.17265504971147
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3368.0,
|
|
"completions/mean_length": 1180.875,
|
|
"completions/mean_terminated_length": 703.8545532226562,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.2370591796934605,
|
|
"epoch": 0.08786848072562359,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2451171875,
|
|
"kl": 0.0010188462001679,
|
|
"learning_rate": 9.126984126984127e-07,
|
|
"loss": 0.0303,
|
|
"num_tokens": 15216191.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4734388589859009,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 155,
|
|
"step_time": 313.3282301053405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3849.0,
|
|
"completions/mean_length": 1148.734375,
|
|
"completions/mean_terminated_length": 843.8448486328125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.053316328674555,
|
|
"epoch": 0.08843537414965986,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.0006997470491114655,
|
|
"learning_rate": 9.121315192743763e-07,
|
|
"loss": -0.0515,
|
|
"num_tokens": 15301862.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4917473793029785,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 156,
|
|
"step_time": 299.08300923835486
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3414.0,
|
|
"completions/mean_length": 1203.828125,
|
|
"completions/mean_terminated_length": 904.637939453125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.421551700681448,
|
|
"epoch": 0.08900226757369614,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.373046875,
|
|
"kl": 0.0007786054011376109,
|
|
"learning_rate": 9.115646258503401e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 15390323.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 157,
|
|
"step_time": 294.19915232714266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3767.0,
|
|
"completions/mean_length": 969.390625,
|
|
"completions/mean_terminated_length": 704.4237060546875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.2527791261672974,
|
|
"epoch": 0.08956916099773243,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.35546875,
|
|
"kl": 0.0009187185551127186,
|
|
"learning_rate": 9.109977324263038e-07,
|
|
"loss": 0.0473,
|
|
"num_tokens": 15462524.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.7014724016189575,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 158,
|
|
"step_time": 290.991029872559
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3310.0,
|
|
"completions/mean_length": 1289.6875,
|
|
"completions/mean_terminated_length": 770.0,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.842462159693241,
|
|
"epoch": 0.09013605442176871,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.296875,
|
|
"kl": 0.0010018895627581514,
|
|
"learning_rate": 9.104308390022676e-07,
|
|
"loss": -0.0134,
|
|
"num_tokens": 15557616.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 159,
|
|
"step_time": 298.8811679929495
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4079.0,
|
|
"completions/mean_length": 1188.234375,
|
|
"completions/mean_terminated_length": 994.3833618164062,
|
|
"completions/min_length": 32.0,
|
|
"completions/min_terminated_length": 32.0,
|
|
"entropy": 4.564862992614508,
|
|
"epoch": 0.09070294784580499,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2138671875,
|
|
"kl": 0.0007492936820199247,
|
|
"learning_rate": 9.098639455782312e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 15644159.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 160,
|
|
"step_time": 291.18783896882087
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3548.0,
|
|
"completions/mean_length": 1427.71875,
|
|
"completions/mean_terminated_length": 933.5925903320312,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.1501113399863243,
|
|
"epoch": 0.09126984126984126,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.109375,
|
|
"kl": 0.0007935078247101046,
|
|
"learning_rate": 9.09297052154195e-07,
|
|
"loss": 0.0065,
|
|
"num_tokens": 15748317.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 161,
|
|
"step_time": 298.3245401792228
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3652.0,
|
|
"completions/mean_length": 1111.59375,
|
|
"completions/mean_terminated_length": 858.677978515625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.9220812749117613,
|
|
"epoch": 0.09183673469387756,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.333984375,
|
|
"kl": 0.0007548401827079942,
|
|
"learning_rate": 9.087301587301587e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 15830043.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 162,
|
|
"step_time": 292.2724645556882
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4003.0,
|
|
"completions/mean_length": 1288.25,
|
|
"completions/mean_terminated_length": 1050.3050537109375,
|
|
"completions/min_length": 35.0,
|
|
"completions/min_terminated_length": 35.0,
|
|
"entropy": 3.6803053356707096,
|
|
"epoch": 0.09240362811791383,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1591796875,
|
|
"kl": 0.001038718895870261,
|
|
"learning_rate": 9.081632653061225e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 15924699.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 163,
|
|
"step_time": 295.85355997178704
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3796.0,
|
|
"completions/mean_length": 1493.40625,
|
|
"completions/mean_terminated_length": 1121.607177734375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.4482744075357914,
|
|
"epoch": 0.09297052154195011,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.296875,
|
|
"kl": 0.0007815584158379352,
|
|
"learning_rate": 9.075963718820861e-07,
|
|
"loss": -0.0798,
|
|
"num_tokens": 16033693.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 164,
|
|
"step_time": 298.8449371922761
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4059.0,
|
|
"completions/mean_length": 1206.765625,
|
|
"completions/mean_terminated_length": 907.8793334960938,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 4.567217536270618,
|
|
"epoch": 0.0935374149659864,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.0009052703826455399,
|
|
"learning_rate": 9.070294784580499e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 16121230.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 165,
|
|
"step_time": 292.9822875568643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3983.0,
|
|
"completions/mean_length": 1307.03125,
|
|
"completions/mean_terminated_length": 850.654541015625,
|
|
"completions/min_length": 38.0,
|
|
"completions/min_terminated_length": 38.0,
|
|
"entropy": 3.0286555476486683,
|
|
"epoch": 0.09410430839002268,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1689453125,
|
|
"kl": 0.001030411476676818,
|
|
"learning_rate": 9.064625850340135e-07,
|
|
"loss": 0.0178,
|
|
"num_tokens": 16217136.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 166,
|
|
"step_time": 304.10015850979835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3961.0,
|
|
"completions/mean_length": 1006.484375,
|
|
"completions/mean_terminated_length": 686.8793334960938,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.243671216070652,
|
|
"epoch": 0.09467120181405896,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2109375,
|
|
"kl": 0.0010857144989131484,
|
|
"learning_rate": 9.058956916099772e-07,
|
|
"loss": 0.0292,
|
|
"num_tokens": 16292823.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6529466509819031,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 167,
|
|
"step_time": 297.90582262165844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3689.0,
|
|
"completions/mean_length": 1202.625,
|
|
"completions/mean_terminated_length": 729.1636352539062,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 2.780062321573496,
|
|
"epoch": 0.09523809523809523,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.166015625,
|
|
"kl": 0.0010662161021173233,
|
|
"learning_rate": 9.05328798185941e-07,
|
|
"loss": 0.0149,
|
|
"num_tokens": 16382351.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 168,
|
|
"step_time": 298.7541268877685
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3988.0,
|
|
"completions/mean_length": 1284.109375,
|
|
"completions/mean_terminated_length": 1096.6500244140625,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"entropy": 4.45481001585722,
|
|
"epoch": 0.09580498866213152,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1748046875,
|
|
"kl": 0.0007455445957020856,
|
|
"learning_rate": 9.047619047619047e-07,
|
|
"loss": 0.0211,
|
|
"num_tokens": 16477158.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 169,
|
|
"step_time": 296.70281307492405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4081.0,
|
|
"completions/mean_length": 1465.640625,
|
|
"completions/mean_terminated_length": 919.7169799804688,
|
|
"completions/min_length": 19.0,
|
|
"completions/min_terminated_length": 19.0,
|
|
"entropy": 3.220314707607031,
|
|
"epoch": 0.0963718820861678,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.09765625,
|
|
"kl": 0.0007882528479967732,
|
|
"learning_rate": 9.041950113378685e-07,
|
|
"loss": -0.0049,
|
|
"num_tokens": 16585927.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 170,
|
|
"step_time": 312.9412234034389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3918.0,
|
|
"completions/mean_length": 1084.859375,
|
|
"completions/mean_terminated_length": 829.677978515625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.584567127749324,
|
|
"epoch": 0.09693877551020408,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.365234375,
|
|
"kl": 0.001218626624904573,
|
|
"learning_rate": 9.036281179138322e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 16665574.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 171,
|
|
"step_time": 291.5702755851671
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3636.0,
|
|
"completions/mean_length": 1258.171875,
|
|
"completions/mean_terminated_length": 909.6666870117188,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.920179232954979,
|
|
"epoch": 0.09750566893424037,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.0009037836898642126,
|
|
"learning_rate": 9.030612244897958e-07,
|
|
"loss": -0.0098,
|
|
"num_tokens": 16757289.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 172,
|
|
"step_time": 296.3245643340051
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4070.0,
|
|
"completions/mean_length": 1590.71875,
|
|
"completions/mean_terminated_length": 1070.7547607421875,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"entropy": 4.072601463645697,
|
|
"epoch": 0.09807256235827665,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2392578125,
|
|
"kl": 0.0006884984322823584,
|
|
"learning_rate": 9.024943310657595e-07,
|
|
"loss": -0.0396,
|
|
"num_tokens": 16870271.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 173,
|
|
"step_time": 292.68767904955894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3377.0,
|
|
"completions/mean_length": 1331.140625,
|
|
"completions/mean_terminated_length": 878.7090454101562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.180808439850807,
|
|
"epoch": 0.09863945578231292,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1689453125,
|
|
"kl": 0.0011846703018818516,
|
|
"learning_rate": 9.019274376417233e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 16967256.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 174,
|
|
"step_time": 296.2339540896937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3701.0,
|
|
"completions/mean_length": 1500.0625,
|
|
"completions/mean_terminated_length": 901.0000610351562,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.574791617691517,
|
|
"epoch": 0.0992063492063492,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2236328125,
|
|
"kl": 0.0007171590723373811,
|
|
"learning_rate": 9.01360544217687e-07,
|
|
"loss": 0.0315,
|
|
"num_tokens": 17077364.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 175,
|
|
"step_time": 315.0234984504059
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3318.0,
|
|
"completions/mean_length": 1103.359375,
|
|
"completions/mean_terminated_length": 793.77587890625,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.082523837685585,
|
|
"epoch": 0.09977324263038549,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.275390625,
|
|
"kl": 0.0009175605064228876,
|
|
"learning_rate": 9.007936507936508e-07,
|
|
"loss": -0.031,
|
|
"num_tokens": 17159283.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 176,
|
|
"step_time": 294.17216272838414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3481.0,
|
|
"completions/mean_length": 1434.796875,
|
|
"completions/mean_terminated_length": 882.4717407226562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.4342534486204386,
|
|
"epoch": 0.10034013605442177,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1396484375,
|
|
"kl": 0.0009482698242209153,
|
|
"learning_rate": 9.002267573696145e-07,
|
|
"loss": 0.006,
|
|
"num_tokens": 17262086.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 177,
|
|
"step_time": 294.06646401435137
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3024.0,
|
|
"completions/mean_length": 1340.296875,
|
|
"completions/mean_terminated_length": 768.3585205078125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.5993550419807434,
|
|
"epoch": 0.10090702947845805,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.177734375,
|
|
"kl": 0.0009339848129457096,
|
|
"learning_rate": 8.996598639455783e-07,
|
|
"loss": 0.0196,
|
|
"num_tokens": 17358305.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 178,
|
|
"step_time": 295.2236841134727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3958.0,
|
|
"completions/mean_length": 1307.96875,
|
|
"completions/mean_terminated_length": 1019.5516967773438,
|
|
"completions/min_length": 64.0,
|
|
"completions/min_terminated_length": 64.0,
|
|
"entropy": 3.9585630483925343,
|
|
"epoch": 0.10147392290249434,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1259765625,
|
|
"kl": 0.0008951632571552182,
|
|
"learning_rate": 8.990929705215419e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 17454799.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 179,
|
|
"step_time": 304.4246026193723
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3604.0,
|
|
"completions/mean_length": 1124.375,
|
|
"completions/mean_terminated_length": 759.4385986328125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.9999672435224056,
|
|
"epoch": 0.10204081632653061,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2119140625,
|
|
"kl": 0.0012945063681399915,
|
|
"learning_rate": 8.985260770975056e-07,
|
|
"loss": 0.0298,
|
|
"num_tokens": 17540831.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 180,
|
|
"step_time": 312.59750212263316
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3975.0,
|
|
"completions/mean_length": 1310.15625,
|
|
"completions/mean_terminated_length": 854.2908935546875,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 2.9158868305385113,
|
|
"epoch": 0.10260770975056689,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.0009765556005731924,
|
|
"learning_rate": 8.979591836734693e-07,
|
|
"loss": 0.0723,
|
|
"num_tokens": 17637145.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 181,
|
|
"step_time": 298.92864027246833
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3849.0,
|
|
"completions/mean_length": 1212.890625,
|
|
"completions/mean_terminated_length": 858.8245849609375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.62555655837059,
|
|
"epoch": 0.10317460317460317,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2890625,
|
|
"kl": 0.0009739097476995084,
|
|
"learning_rate": 8.973922902494331e-07,
|
|
"loss": 0.0806,
|
|
"num_tokens": 17725850.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 182,
|
|
"step_time": 293.8928272379562
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4081.0,
|
|
"completions/mean_length": 1504.3125,
|
|
"completions/mean_terminated_length": 1080.2181396484375,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.831495199352503,
|
|
"epoch": 0.10374149659863946,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.0009495031881670002,
|
|
"learning_rate": 8.968253968253968e-07,
|
|
"loss": -0.0043,
|
|
"num_tokens": 17833318.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 183,
|
|
"step_time": 295.0573527580127
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4065.0,
|
|
"completions/mean_length": 1112.140625,
|
|
"completions/mean_terminated_length": 745.7017822265625,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 2.254999879747629,
|
|
"epoch": 0.10430839002267574,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.0011347034851496574,
|
|
"learning_rate": 8.962585034013606e-07,
|
|
"loss": -0.1194,
|
|
"num_tokens": 17915535.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 184,
|
|
"step_time": 292.29882525559515
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3741.0,
|
|
"completions/mean_length": 1387.078125,
|
|
"completions/mean_terminated_length": 943.7999877929688,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 3.113091289997101,
|
|
"epoch": 0.10487528344671201,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1357421875,
|
|
"kl": 0.000856978060255642,
|
|
"learning_rate": 8.956916099773242e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 18015060.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 185,
|
|
"step_time": 295.9827573513612
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3272.0,
|
|
"completions/mean_length": 1304.546875,
|
|
"completions/mean_terminated_length": 787.6111450195312,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 3.156414870172739,
|
|
"epoch": 0.1054421768707483,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.00018310546875,
|
|
"kl": 0.0009036011124408105,
|
|
"learning_rate": 8.95124716553288e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 18111255.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 186,
|
|
"step_time": 300.288443938829
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3863.0,
|
|
"completions/mean_length": 1590.90625,
|
|
"completions/mean_terminated_length": 1180.9818115234375,
|
|
"completions/min_length": 145.0,
|
|
"completions/min_terminated_length": 145.0,
|
|
"entropy": 3.8824569322168827,
|
|
"epoch": 0.10600907029478458,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.0007456481725967024,
|
|
"learning_rate": 8.945578231292517e-07,
|
|
"loss": 0.0122,
|
|
"num_tokens": 18224001.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 187,
|
|
"step_time": 292.1030384255573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3740.0,
|
|
"completions/mean_length": 1235.4375,
|
|
"completions/mean_terminated_length": 826.7857666015625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.6696518138051033,
|
|
"epoch": 0.10657596371882086,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2001953125,
|
|
"kl": 0.0008291571648442186,
|
|
"learning_rate": 8.939909297052153e-07,
|
|
"loss": 0.0292,
|
|
"num_tokens": 18314429.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 188,
|
|
"step_time": 293.7575126942247
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3709.0,
|
|
"completions/mean_length": 1473.890625,
|
|
"completions/mean_terminated_length": 988.3148193359375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.7852463982999325,
|
|
"epoch": 0.10714285714285714,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.111328125,
|
|
"kl": 0.0007867878121032845,
|
|
"learning_rate": 8.934240362811791e-07,
|
|
"loss": 0.0176,
|
|
"num_tokens": 18421734.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 189,
|
|
"step_time": 298.22808313928545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3318.0,
|
|
"completions/mean_length": 1484.609375,
|
|
"completions/mean_terminated_length": 942.6226806640625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.4584941156208515,
|
|
"epoch": 0.10770975056689343,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2021484375,
|
|
"kl": 0.0011089006511610933,
|
|
"learning_rate": 8.928571428571428e-07,
|
|
"loss": 0.0267,
|
|
"num_tokens": 18527917.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 190,
|
|
"step_time": 296.6196203548461
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3976.0,
|
|
"completions/mean_length": 1231.890625,
|
|
"completions/mean_terminated_length": 822.732177734375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.165639989078045,
|
|
"epoch": 0.1082766439909297,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.34375,
|
|
"kl": 0.0011495280305098277,
|
|
"learning_rate": 8.922902494331065e-07,
|
|
"loss": 0.0808,
|
|
"num_tokens": 18620350.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 191,
|
|
"step_time": 300.34041826426983
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4074.0,
|
|
"completions/mean_length": 1468.296875,
|
|
"completions/mean_terminated_length": 1092.9107666015625,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"entropy": 4.020199842751026,
|
|
"epoch": 0.10884353741496598,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.09033203125,
|
|
"kl": 0.0010856879234779626,
|
|
"learning_rate": 8.917233560090702e-07,
|
|
"loss": -0.0153,
|
|
"num_tokens": 18725657.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 192,
|
|
"step_time": 295.3089788649231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3248.0,
|
|
"completions/mean_length": 949.515625,
|
|
"completions/mean_terminated_length": 739.7500610351562,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.225232597440481,
|
|
"epoch": 0.10941043083900227,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.0010666561047401046,
|
|
"learning_rate": 8.91156462585034e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 18797514.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 193,
|
|
"step_time": 296.91239412501454
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2699.0,
|
|
"completions/mean_length": 1026.953125,
|
|
"completions/mean_terminated_length": 766.8643798828125,
|
|
"completions/min_length": 139.0,
|
|
"completions/min_terminated_length": 139.0,
|
|
"entropy": 1.918291261419654,
|
|
"epoch": 0.10997732426303855,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2080078125,
|
|
"kl": 0.000961602780080284,
|
|
"learning_rate": 8.905895691609977e-07,
|
|
"loss": -0.0118,
|
|
"num_tokens": 18873423.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 194,
|
|
"step_time": 292.62212283257395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3505.0,
|
|
"completions/mean_length": 1220.703125,
|
|
"completions/mean_terminated_length": 1029.0167236328125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.112483162432909,
|
|
"epoch": 0.11054421768707483,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2177734375,
|
|
"kl": 0.0008761044791754102,
|
|
"learning_rate": 8.900226757369615e-07,
|
|
"loss": 0.0674,
|
|
"num_tokens": 18962124.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 195,
|
|
"step_time": 294.0724186487496
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3981.0,
|
|
"completions/mean_length": 1406.90625,
|
|
"completions/mean_terminated_length": 966.8726806640625,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 3.886863924562931,
|
|
"epoch": 0.1111111111111111,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.0009999317062465707,
|
|
"learning_rate": 8.894557823129251e-07,
|
|
"loss": 0.0176,
|
|
"num_tokens": 19067294.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 196,
|
|
"step_time": 326.1539095537737
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3863.0,
|
|
"completions/mean_length": 1156.734375,
|
|
"completions/mean_terminated_length": 736.8392944335938,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.956690076738596,
|
|
"epoch": 0.1116780045351474,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.251953125,
|
|
"kl": 0.0012019378827972105,
|
|
"learning_rate": 8.888888888888888e-07,
|
|
"loss": 0.0566,
|
|
"num_tokens": 19154893.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 197,
|
|
"step_time": 301.8200301043689
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3912.0,
|
|
"completions/mean_length": 1050.71875,
|
|
"completions/mean_terminated_length": 735.6896362304688,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.1997528932988644,
|
|
"epoch": 0.11224489795918367,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1826171875,
|
|
"kl": 0.0012252896412974223,
|
|
"learning_rate": 8.883219954648525e-07,
|
|
"loss": 0.0173,
|
|
"num_tokens": 19234467.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5092765688896179,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 198,
|
|
"step_time": 302.63945909496397
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4051.0,
|
|
"completions/mean_length": 1176.40625,
|
|
"completions/mean_terminated_length": 874.3793334960938,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.246236026287079,
|
|
"epoch": 0.11281179138321995,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.0011566742414288456,
|
|
"learning_rate": 8.877551020408163e-07,
|
|
"loss": 0.0341,
|
|
"num_tokens": 19320229.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 199,
|
|
"step_time": 294.37082410696894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4041.0,
|
|
"completions/mean_length": 1260.75,
|
|
"completions/mean_terminated_length": 796.7999877929688,
|
|
"completions/min_length": 75.0,
|
|
"completions/min_terminated_length": 75.0,
|
|
"entropy": 3.445892170071602,
|
|
"epoch": 0.11337868480725624,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.310546875,
|
|
"kl": 0.0012227596816956066,
|
|
"learning_rate": 8.8718820861678e-07,
|
|
"loss": -0.0124,
|
|
"num_tokens": 19412173.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 200,
|
|
"step_time": 293.2042292887345
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3265.0,
|
|
"completions/mean_length": 1124.421875,
|
|
"completions/mean_terminated_length": 759.4912109375,
|
|
"completions/min_length": 30.0,
|
|
"completions/min_terminated_length": 30.0,
|
|
"entropy": 2.0993868820369244,
|
|
"epoch": 0.11394557823129252,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2294921875,
|
|
"kl": 0.0008618845185992541,
|
|
"learning_rate": 8.866213151927438e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 19495376.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 201,
|
|
"step_time": 296.0682121384889
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4012.0,
|
|
"completions/mean_length": 1554.3125,
|
|
"completions/mean_terminated_length": 906.431396484375,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.8276742175221443,
|
|
"epoch": 0.1145124716553288,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.13671875,
|
|
"kl": 0.0011864844018418808,
|
|
"learning_rate": 8.860544217687075e-07,
|
|
"loss": -0.0066,
|
|
"num_tokens": 19608212.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 202,
|
|
"step_time": 312.8659586533904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3812.0,
|
|
"completions/mean_length": 1436.359375,
|
|
"completions/mean_terminated_length": 1001.1454467773438,
|
|
"completions/min_length": 51.0,
|
|
"completions/min_terminated_length": 51.0,
|
|
"entropy": 4.753630865365267,
|
|
"epoch": 0.11507936507936507,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.154296875,
|
|
"kl": 0.0015447768164449371,
|
|
"learning_rate": 8.854875283446712e-07,
|
|
"loss": 0.0476,
|
|
"num_tokens": 19710979.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 203,
|
|
"step_time": 294.42433897126466
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3775.0,
|
|
"completions/mean_length": 1548.28125,
|
|
"completions/mean_terminated_length": 1076.4814453125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.073138736188412,
|
|
"epoch": 0.11564625850340136,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2138671875,
|
|
"kl": 0.0007871567868278362,
|
|
"learning_rate": 8.849206349206348e-07,
|
|
"loss": 0.0537,
|
|
"num_tokens": 19821317.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 204,
|
|
"step_time": 295.367243854329
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3729.0,
|
|
"completions/mean_length": 1307.0,
|
|
"completions/mean_terminated_length": 908.5714721679688,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.909986387938261,
|
|
"epoch": 0.11621315192743764,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1982421875,
|
|
"kl": 0.0007760782882542117,
|
|
"learning_rate": 8.843537414965985e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 19915189.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 205,
|
|
"step_time": 291.66046211402863
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3515.0,
|
|
"completions/mean_length": 1305.734375,
|
|
"completions/mean_terminated_length": 789.0185546875,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 2.36519917473197,
|
|
"epoch": 0.11678004535147392,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1796875,
|
|
"kl": 0.0011316735999571392,
|
|
"learning_rate": 8.837868480725623e-07,
|
|
"loss": 0.0065,
|
|
"num_tokens": 20011828.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 206,
|
|
"step_time": 299.7828513002023
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4080.0,
|
|
"completions/mean_length": 881.109375,
|
|
"completions/mean_terminated_length": 830.0794067382812,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 3.779766697436571,
|
|
"epoch": 0.11734693877551021,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.0010524408498895355,
|
|
"learning_rate": 8.83219954648526e-07,
|
|
"loss": 0.0453,
|
|
"num_tokens": 20080899.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.8538762331008911,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 207,
|
|
"step_time": 308.67669108323753
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3160.0,
|
|
"completions/mean_length": 1045.453125,
|
|
"completions/mean_terminated_length": 842.0833740234375,
|
|
"completions/min_length": 147.0,
|
|
"completions/min_terminated_length": 147.0,
|
|
"entropy": 3.642435859888792,
|
|
"epoch": 0.11791383219954649,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30078125,
|
|
"kl": 0.0010334904181945603,
|
|
"learning_rate": 8.826530612244898e-07,
|
|
"loss": 0.0998,
|
|
"num_tokens": 20158808.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 208,
|
|
"step_time": 293.9838742958382
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3693.0,
|
|
"completions/mean_length": 1332.265625,
|
|
"completions/mean_terminated_length": 880.0181274414062,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.808041248470545,
|
|
"epoch": 0.11848072562358276,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.15234375,
|
|
"kl": 0.0011558761889318703,
|
|
"learning_rate": 8.820861678004535e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 20254625.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 209,
|
|
"step_time": 291.6761906463653
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4050.0,
|
|
"completions/mean_length": 1218.421875,
|
|
"completions/mean_terminated_length": 974.559326171875,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"entropy": 3.6133025735616684,
|
|
"epoch": 0.11904761904761904,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.224609375,
|
|
"kl": 0.0008826196917652851,
|
|
"learning_rate": 8.815192743764172e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 20343972.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 210,
|
|
"step_time": 292.4662137590349
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3527.0,
|
|
"completions/mean_length": 1256.0,
|
|
"completions/mean_terminated_length": 532.0784301757812,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.478566935285926,
|
|
"epoch": 0.11961451247165533,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.0017366717056574998,
|
|
"learning_rate": 8.809523809523809e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 20435484.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 211,
|
|
"step_time": 297.2094409316778
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3968.0,
|
|
"completions/mean_length": 1577.015625,
|
|
"completions/mean_terminated_length": 1110.5369873046875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.70728861913085,
|
|
"epoch": 0.12018140589569161,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.146484375,
|
|
"kl": 0.0008267538469226565,
|
|
"learning_rate": 8.803854875283446e-07,
|
|
"loss": 0.028,
|
|
"num_tokens": 20547469.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 212,
|
|
"step_time": 295.8759975992143
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3732.0,
|
|
"completions/mean_length": 1148.28125,
|
|
"completions/mean_terminated_length": 843.3448486328125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.0519875772297382,
|
|
"epoch": 0.12074829931972789,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.234375,
|
|
"kl": 0.0012846419031120604,
|
|
"learning_rate": 8.798185941043083e-07,
|
|
"loss": 0.0341,
|
|
"num_tokens": 20633791.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.41539984941482544,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 213,
|
|
"step_time": 298.65141285955906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4010.0,
|
|
"completions/mean_length": 1745.609375,
|
|
"completions/mean_terminated_length": 1087.5,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.023875005543232,
|
|
"epoch": 0.12131519274376418,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.138671875,
|
|
"kl": 0.0007914034013083437,
|
|
"learning_rate": 8.792517006802721e-07,
|
|
"loss": 0.0095,
|
|
"num_tokens": 20757390.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5579067468643188,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 214,
|
|
"step_time": 299.87372230086476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3233.0,
|
|
"completions/mean_length": 976.15625,
|
|
"completions/mean_terminated_length": 711.7626953125,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 2.71496070548892,
|
|
"epoch": 0.12188208616780045,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2451171875,
|
|
"kl": 0.0010727268054324668,
|
|
"learning_rate": 8.786848072562358e-07,
|
|
"loss": 0.0041,
|
|
"num_tokens": 20831504.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 215,
|
|
"step_time": 298.7091171545908
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3868.0,
|
|
"completions/mean_length": 1516.609375,
|
|
"completions/mean_terminated_length": 794.3800048828125,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 3.812237862497568,
|
|
"epoch": 0.12244897959183673,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.203125,
|
|
"kl": 0.0013310160175024066,
|
|
"learning_rate": 8.781179138321995e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 20944471.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 216,
|
|
"step_time": 313.4162803757936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3581.0,
|
|
"completions/mean_length": 1257.0625,
|
|
"completions/mean_terminated_length": 1117.4425048828125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.202047243714333,
|
|
"epoch": 0.12301587301587301,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.28125,
|
|
"kl": 0.0007741837871435564,
|
|
"learning_rate": 8.775510204081632e-07,
|
|
"loss": 0.0311,
|
|
"num_tokens": 21036235.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 217,
|
|
"step_time": 300.48187935724854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3732.0,
|
|
"completions/mean_length": 1682.34375,
|
|
"completions/mean_terminated_length": 1287.3818359375,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 4.5055681094527245,
|
|
"epoch": 0.1235827664399093,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.000785970254582935,
|
|
"learning_rate": 8.76984126984127e-07,
|
|
"loss": 0.0282,
|
|
"num_tokens": 21155937.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 218,
|
|
"step_time": 299.3899869751185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4067.0,
|
|
"completions/mean_length": 1030.34375,
|
|
"completions/mean_terminated_length": 825.9667358398438,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.4379561692476273,
|
|
"epoch": 0.12414965986394558,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2412109375,
|
|
"kl": 0.0011134496562590357,
|
|
"learning_rate": 8.764172335600907e-07,
|
|
"loss": 0.0166,
|
|
"num_tokens": 21233175.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 219,
|
|
"step_time": 292.3021672721952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3961.0,
|
|
"completions/mean_length": 1548.75,
|
|
"completions/mean_terminated_length": 1020.0755004882812,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 3.782156143337488,
|
|
"epoch": 0.12471655328798185,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1396484375,
|
|
"kl": 0.001276508145565458,
|
|
"learning_rate": 8.758503401360544e-07,
|
|
"loss": -0.0023,
|
|
"num_tokens": 21344287.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 220,
|
|
"step_time": 298.39621212985367
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4026.0,
|
|
"completions/mean_length": 1192.40625,
|
|
"completions/mean_terminated_length": 998.8333740234375,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.345248498022556,
|
|
"epoch": 0.12528344671201813,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.177734375,
|
|
"kl": 0.0008278152436105302,
|
|
"learning_rate": 8.752834467120181e-07,
|
|
"loss": 0.0474,
|
|
"num_tokens": 21432529.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 221,
|
|
"step_time": 294.27885660901666
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3505.0,
|
|
"completions/mean_length": 1050.15625,
|
|
"completions/mean_terminated_length": 792.0338745117188,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.82575561478734,
|
|
"epoch": 0.12585034013605442,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.333984375,
|
|
"kl": 0.0010389726776338648,
|
|
"learning_rate": 8.747165532879818e-07,
|
|
"loss": -0.0022,
|
|
"num_tokens": 21513307.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 222,
|
|
"step_time": 312.0486962515861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3608.0,
|
|
"completions/mean_length": 1200.875,
|
|
"completions/mean_terminated_length": 845.3333129882812,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.7446229569613934,
|
|
"epoch": 0.1264172335600907,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.310546875,
|
|
"kl": 0.001183695454528788,
|
|
"learning_rate": 8.741496598639455e-07,
|
|
"loss": 0.0173,
|
|
"num_tokens": 21601027.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 223,
|
|
"step_time": 293.27236039657146
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4052.0,
|
|
"completions/mean_length": 1081.3125,
|
|
"completions/mean_terminated_length": 825.8305053710938,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 2.898800101131201,
|
|
"epoch": 0.12698412698412698,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2412109375,
|
|
"kl": 0.0013538178318412974,
|
|
"learning_rate": 8.735827664399093e-07,
|
|
"loss": 0.0675,
|
|
"num_tokens": 21681167.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 224,
|
|
"step_time": 292.89520471170545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3207.0,
|
|
"completions/mean_length": 1052.921875,
|
|
"completions/mean_terminated_length": 738.1206665039062,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 3.210470274090767,
|
|
"epoch": 0.12755102040816327,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.0011044722068618285,
|
|
"learning_rate": 8.73015873015873e-07,
|
|
"loss": 0.0513,
|
|
"num_tokens": 21765786.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.8020543456077576,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 225,
|
|
"step_time": 337.17833896446973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4095.0,
|
|
"completions/mean_length": 924.1875,
|
|
"completions/mean_terminated_length": 712.7333984375,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"entropy": 2.6770739629864693,
|
|
"epoch": 0.12811791383219956,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.0011462787570053479,
|
|
"learning_rate": 8.724489795918368e-07,
|
|
"loss": 0.0218,
|
|
"num_tokens": 21835750.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 226,
|
|
"step_time": 294.12168524228036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3575.0,
|
|
"completions/mean_length": 1203.609375,
|
|
"completions/mean_terminated_length": 790.4107666015625,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 3.102189105004072,
|
|
"epoch": 0.12868480725623582,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.169921875,
|
|
"kl": 0.001319015835179016,
|
|
"learning_rate": 8.718820861678005e-07,
|
|
"loss": 0.0073,
|
|
"num_tokens": 21923997.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 227,
|
|
"step_time": 296.67299195192754
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4084.0,
|
|
"completions/mean_length": 1021.75,
|
|
"completions/mean_terminated_length": 870.5573120117188,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.400593936443329,
|
|
"epoch": 0.1292517006802721,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.224609375,
|
|
"kl": 0.001097561624192167,
|
|
"learning_rate": 8.71315192743764e-07,
|
|
"loss": -0.002,
|
|
"num_tokens": 22000061.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 228,
|
|
"step_time": 292.2873935243115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3552.0,
|
|
"completions/mean_length": 1350.84375,
|
|
"completions/mean_terminated_length": 958.6785888671875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.85386623442173,
|
|
"epoch": 0.12981859410430838,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.0013140782157279318,
|
|
"learning_rate": 8.707482993197278e-07,
|
|
"loss": 0.0063,
|
|
"num_tokens": 22097731.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 229,
|
|
"step_time": 295.80912455078214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3852.0,
|
|
"completions/mean_length": 1410.8125,
|
|
"completions/mean_terminated_length": 913.5555419921875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.6953197680413723,
|
|
"epoch": 0.13038548752834467,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.154296875,
|
|
"kl": 0.0011070836553699337,
|
|
"learning_rate": 8.701814058956915e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 22199487.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 230,
|
|
"step_time": 296.0781445270404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4094.0,
|
|
"completions/mean_length": 1486.34375,
|
|
"completions/mean_terminated_length": 1003.0740966796875,
|
|
"completions/min_length": 36.0,
|
|
"completions/min_terminated_length": 36.0,
|
|
"entropy": 3.8559432066977024,
|
|
"epoch": 0.13095238095238096,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.001084855654880812,
|
|
"learning_rate": 8.696145124716553e-07,
|
|
"loss": 0.0235,
|
|
"num_tokens": 22304917.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 231,
|
|
"step_time": 294.1224720198661
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3586.0,
|
|
"completions/mean_length": 1244.890625,
|
|
"completions/mean_terminated_length": 894.75439453125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.2090540044009686,
|
|
"epoch": 0.13151927437641722,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.0009592715159669751,
|
|
"learning_rate": 8.69047619047619e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 22396678.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 232,
|
|
"step_time": 299.46676432620734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3794.0,
|
|
"completions/mean_length": 976.46875,
|
|
"completions/mean_terminated_length": 653.7586059570312,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"entropy": 3.0212653540074825,
|
|
"epoch": 0.1320861678004535,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1943359375,
|
|
"kl": 0.0018015931818808895,
|
|
"learning_rate": 8.684807256235828e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 22470708.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 233,
|
|
"step_time": 294.826646476984
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4034.0,
|
|
"completions/mean_length": 1293.921875,
|
|
"completions/mean_terminated_length": 1004.0516967773438,
|
|
"completions/min_length": 49.0,
|
|
"completions/min_terminated_length": 49.0,
|
|
"entropy": 3.842056378722191,
|
|
"epoch": 0.1326530612244898,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.25390625,
|
|
"kl": 0.0010123011907126056,
|
|
"learning_rate": 8.679138321995464e-07,
|
|
"loss": -0.0169,
|
|
"num_tokens": 22566031.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 234,
|
|
"step_time": 304.18481398466974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3868.0,
|
|
"completions/mean_length": 1329.65625,
|
|
"completions/mean_terminated_length": 817.370361328125,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 3.2029078900814056,
|
|
"epoch": 0.13321995464852607,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1357421875,
|
|
"kl": 0.0013659713295055553,
|
|
"learning_rate": 8.673469387755102e-07,
|
|
"loss": 0.0055,
|
|
"num_tokens": 22667241.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 235,
|
|
"step_time": 315.4635110665113
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3624.0,
|
|
"completions/mean_length": 1535.328125,
|
|
"completions/mean_terminated_length": 1061.129638671875,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 2.929383337497711,
|
|
"epoch": 0.13378684807256236,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1787109375,
|
|
"kl": 0.0010227088023384567,
|
|
"learning_rate": 8.667800453514738e-07,
|
|
"loss": 0.0159,
|
|
"num_tokens": 22780798.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 236,
|
|
"step_time": 308.49995029717684
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3946.0,
|
|
"completions/mean_length": 1041.796875,
|
|
"completions/mean_terminated_length": 838.183349609375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.236738458275795,
|
|
"epoch": 0.13435374149659865,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.0012075366357748862,
|
|
"learning_rate": 8.662131519274376e-07,
|
|
"loss": 0.0624,
|
|
"num_tokens": 22859321.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 237,
|
|
"step_time": 300.28507947269827
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4058.0,
|
|
"completions/mean_length": 1361.09375,
|
|
"completions/mean_terminated_length": 913.5635986328125,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.180329939350486,
|
|
"epoch": 0.1349206349206349,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2265625,
|
|
"kl": 0.0014698277136631077,
|
|
"learning_rate": 8.656462585034013e-07,
|
|
"loss": 0.0046,
|
|
"num_tokens": 22957831.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 238,
|
|
"step_time": 294.9036829229444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3610.0,
|
|
"completions/mean_length": 1153.0,
|
|
"completions/mean_terminated_length": 903.5932006835938,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.6870043724775314,
|
|
"epoch": 0.1354875283446712,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1591796875,
|
|
"kl": 0.0015672321533202194,
|
|
"learning_rate": 8.650793650793651e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 23042711.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 239,
|
|
"step_time": 295.62402374856174
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3883.0,
|
|
"completions/mean_length": 1244.921875,
|
|
"completions/mean_terminated_length": 837.6250610351562,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 3.7111910469830036,
|
|
"epoch": 0.1360544217687075,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.154296875,
|
|
"kl": 0.0011819430092145922,
|
|
"learning_rate": 8.645124716553288e-07,
|
|
"loss": 0.0069,
|
|
"num_tokens": 23133466.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 240,
|
|
"step_time": 295.11580411344767
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4088.0,
|
|
"completions/mean_length": 1177.9375,
|
|
"completions/mean_terminated_length": 761.0714721679688,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.964115008711815,
|
|
"epoch": 0.13662131519274376,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2236328125,
|
|
"kl": 0.0012315598942223005,
|
|
"learning_rate": 8.639455782312925e-07,
|
|
"loss": -0.0057,
|
|
"num_tokens": 23220238.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 241,
|
|
"step_time": 296.22310270369053
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3904.0,
|
|
"completions/mean_length": 1137.546875,
|
|
"completions/mean_terminated_length": 831.5,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.6043158210814,
|
|
"epoch": 0.13718820861678005,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.11962890625,
|
|
"kl": 0.0013280221428431105,
|
|
"learning_rate": 8.633786848072562e-07,
|
|
"loss": -0.0033,
|
|
"num_tokens": 23304257.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.4519989788532257,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 242,
|
|
"step_time": 296.49121211096644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2450.0,
|
|
"completions/mean_length": 911.875,
|
|
"completions/mean_terminated_length": 582.4827270507812,
|
|
"completions/min_length": 27.0,
|
|
"completions/min_terminated_length": 27.0,
|
|
"entropy": 2.447322765365243,
|
|
"epoch": 0.1377551020408163,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.375,
|
|
"kl": 0.002247496593554388,
|
|
"learning_rate": 8.6281179138322e-07,
|
|
"loss": -0.0561,
|
|
"num_tokens": 23374905.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.5076302886009216,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 243,
|
|
"step_time": 301.67275542300195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3944.0,
|
|
"completions/mean_length": 1379.09375,
|
|
"completions/mean_terminated_length": 990.96435546875,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 3.6930803544819355,
|
|
"epoch": 0.1383219954648526,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2138671875,
|
|
"kl": 0.0011230875043111155,
|
|
"learning_rate": 8.622448979591836e-07,
|
|
"loss": 0.0147,
|
|
"num_tokens": 23474367.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5890451669692993,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 244,
|
|
"step_time": 293.819984880276
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3930.0,
|
|
"completions/mean_length": 1300.984375,
|
|
"completions/mean_terminated_length": 843.6181640625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.570481050759554,
|
|
"epoch": 0.1388888888888889,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.251953125,
|
|
"kl": 0.0012484762719395803,
|
|
"learning_rate": 8.616780045351474e-07,
|
|
"loss": 0.0091,
|
|
"num_tokens": 23571702.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.47709178924560547,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 245,
|
|
"step_time": 321.6608476731926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3975.0,
|
|
"completions/mean_length": 1374.109375,
|
|
"completions/mean_terminated_length": 928.7090454101562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.8169440925121307,
|
|
"epoch": 0.13945578231292516,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.001357796514639631,
|
|
"learning_rate": 8.611111111111111e-07,
|
|
"loss": -0.016,
|
|
"num_tokens": 23673061.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 246,
|
|
"step_time": 299.5307418331504
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4082.0,
|
|
"completions/mean_length": 1398.5,
|
|
"completions/mean_terminated_length": 898.9629516601562,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.31706964597106,
|
|
"epoch": 0.14002267573696145,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.0010165272087760968,
|
|
"learning_rate": 8.605442176870747e-07,
|
|
"loss": -0.0157,
|
|
"num_tokens": 23778381.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6281666159629822,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 247,
|
|
"step_time": 323.87792470585555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3988.0,
|
|
"completions/mean_length": 858.96875,
|
|
"completions/mean_terminated_length": 754.54833984375,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 2.975732732564211,
|
|
"epoch": 0.14058956916099774,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2890625,
|
|
"kl": 0.0014932136873540003,
|
|
"learning_rate": 8.599773242630385e-07,
|
|
"loss": 0.082,
|
|
"num_tokens": 23843507.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 248,
|
|
"step_time": 296.37086434010416
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4060.0,
|
|
"completions/mean_length": 1291.453125,
|
|
"completions/mean_terminated_length": 1104.4833984375,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.400741457939148,
|
|
"epoch": 0.141156462585034,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.16796875,
|
|
"kl": 0.0010249239840049995,
|
|
"learning_rate": 8.594104308390022e-07,
|
|
"loss": -0.0014,
|
|
"num_tokens": 23939296.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.32732683420181274,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 249,
|
|
"step_time": 302.395604101941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3825.0,
|
|
"completions/mean_length": 1397.15625,
|
|
"completions/mean_terminated_length": 1011.607177734375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 4.738048255443573,
|
|
"epoch": 0.1417233560090703,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1904296875,
|
|
"kl": 0.001050947124895174,
|
|
"learning_rate": 8.58843537414966e-07,
|
|
"loss": 0.0179,
|
|
"num_tokens": 24040082.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 250,
|
|
"step_time": 295.45087849814445
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3997.0,
|
|
"completions/mean_length": 1354.984375,
|
|
"completions/mean_terminated_length": 963.4107666015625,
|
|
"completions/min_length": 76.0,
|
|
"completions/min_terminated_length": 76.0,
|
|
"entropy": 4.9353371784091,
|
|
"epoch": 0.14229024943310659,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1611328125,
|
|
"kl": 0.0012083352667104919,
|
|
"learning_rate": 8.582766439909297e-07,
|
|
"loss": 0.019,
|
|
"num_tokens": 24139025.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 251,
|
|
"step_time": 298.3130410639569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3563.0,
|
|
"completions/mean_length": 1210.84375,
|
|
"completions/mean_terminated_length": 676.5555419921875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7371104396879673,
|
|
"epoch": 0.14285714285714285,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2333984375,
|
|
"kl": 0.0015491756021219771,
|
|
"learning_rate": 8.577097505668934e-07,
|
|
"loss": 0.0058,
|
|
"num_tokens": 24228535.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 252,
|
|
"step_time": 294.7551212403923
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4043.0,
|
|
"completions/mean_length": 804.984375,
|
|
"completions/mean_terminated_length": 752.74609375,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 2.979743916541338,
|
|
"epoch": 0.14342403628117914,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.400390625,
|
|
"kl": 0.0013989633553137537,
|
|
"learning_rate": 8.57142857142857e-07,
|
|
"loss": 0.057,
|
|
"num_tokens": 24290910.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6154024600982666,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 253,
|
|
"step_time": 293.3673425382003
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3405.0,
|
|
"completions/mean_length": 1081.078125,
|
|
"completions/mean_terminated_length": 769.1896362304688,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 2.687511883676052,
|
|
"epoch": 0.14399092970521543,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.0014965555710659828,
|
|
"learning_rate": 8.565759637188208e-07,
|
|
"loss": 0.0324,
|
|
"num_tokens": 24372307.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 254,
|
|
"step_time": 297.3848839784041
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3124.0,
|
|
"completions/mean_length": 1213.203125,
|
|
"completions/mean_terminated_length": 614.8867797851562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.7044095527380705,
|
|
"epoch": 0.1445578231292517,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.166015625,
|
|
"kl": 0.0013742621849814896,
|
|
"learning_rate": 8.560090702947845e-07,
|
|
"loss": -0.0254,
|
|
"num_tokens": 24461304.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 255,
|
|
"step_time": 294.57220361568034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3713.0,
|
|
"completions/mean_length": 1023.65625,
|
|
"completions/mean_terminated_length": 705.8275756835938,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.3083490654826164,
|
|
"epoch": 0.14512471655328799,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2197265625,
|
|
"kl": 0.0014701982800033875,
|
|
"learning_rate": 8.554421768707483e-07,
|
|
"loss": 0.0396,
|
|
"num_tokens": 24540786.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 256,
|
|
"step_time": 314.96845386736095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2654.0,
|
|
"completions/mean_length": 1142.859375,
|
|
"completions/mean_terminated_length": 659.6181640625,
|
|
"completions/min_length": 174.0,
|
|
"completions/min_terminated_length": 174.0,
|
|
"entropy": 2.601846519857645,
|
|
"epoch": 0.14569160997732428,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.0011016668959200615,
|
|
"learning_rate": 8.54875283446712e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 24627689.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 257,
|
|
"step_time": 312.0238794479519
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3674.0,
|
|
"completions/mean_length": 1136.515625,
|
|
"completions/mean_terminated_length": 990.9671630859375,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 4.248551677912474,
|
|
"epoch": 0.14625850340136054,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.31640625,
|
|
"kl": 0.0011931121989618987,
|
|
"learning_rate": 8.543083900226758e-07,
|
|
"loss": 0.0525,
|
|
"num_tokens": 24710786.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 258,
|
|
"step_time": 291.4342953702435
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3763.0,
|
|
"completions/mean_length": 1476.3125,
|
|
"completions/mean_terminated_length": 808.549072265625,
|
|
"completions/min_length": 101.0,
|
|
"completions/min_terminated_length": 101.0,
|
|
"entropy": 2.9006321597844362,
|
|
"epoch": 0.14682539682539683,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1923828125,
|
|
"kl": 0.0013805588678224012,
|
|
"learning_rate": 8.537414965986394e-07,
|
|
"loss": 0.008,
|
|
"num_tokens": 24817006.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 259,
|
|
"step_time": 302.66142302937806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3869.0,
|
|
"completions/mean_length": 1068.75,
|
|
"completions/mean_terminated_length": 812.203369140625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.3819946944713593,
|
|
"epoch": 0.1473922902494331,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.373046875,
|
|
"kl": 0.0014789492888667155,
|
|
"learning_rate": 8.531746031746031e-07,
|
|
"loss": 0.0723,
|
|
"num_tokens": 24895974.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6986381411552429,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 260,
|
|
"step_time": 296.31661470234394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3818.0,
|
|
"completions/mean_length": 957.65625,
|
|
"completions/mean_terminated_length": 803.3114013671875,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"entropy": 3.298921812325716,
|
|
"epoch": 0.14795918367346939,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.30859375,
|
|
"kl": 0.001135968590460834,
|
|
"learning_rate": 8.526077097505668e-07,
|
|
"loss": 0.0214,
|
|
"num_tokens": 24969304.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 261,
|
|
"step_time": 301.2923899097368
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3981.0,
|
|
"completions/mean_length": 1562.875,
|
|
"completions/mean_terminated_length": 1251.7894287109375,
|
|
"completions/min_length": 144.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"entropy": 3.8599331825971603,
|
|
"epoch": 0.14852607709750568,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.0009275648299080785,
|
|
"learning_rate": 8.520408163265306e-07,
|
|
"loss": 0.0194,
|
|
"num_tokens": 25082000.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 262,
|
|
"step_time": 300.24944328609854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3120.0,
|
|
"completions/mean_length": 1391.6875,
|
|
"completions/mean_terminated_length": 767.6154174804688,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 3.442591927945614,
|
|
"epoch": 0.14909297052154194,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.15234375,
|
|
"kl": 0.0016389939119108021,
|
|
"learning_rate": 8.514739229024943e-07,
|
|
"loss": 0.028,
|
|
"num_tokens": 25182972.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 263,
|
|
"step_time": 293.67484748456627
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3836.0,
|
|
"completions/mean_length": 1541.796875,
|
|
"completions/mean_terminated_length": 826.6199951171875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.9540892839431763,
|
|
"epoch": 0.14965986394557823,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.001244919574673986,
|
|
"learning_rate": 8.509070294784581e-07,
|
|
"loss": -0.0043,
|
|
"num_tokens": 25293119.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 264,
|
|
"step_time": 294.5663408571854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3745.0,
|
|
"completions/mean_length": 1096.5625,
|
|
"completions/mean_terminated_length": 728.2105102539062,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 3.3549214117228985,
|
|
"epoch": 0.15022675736961452,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2392578125,
|
|
"kl": 0.0013892023780499585,
|
|
"learning_rate": 8.503401360544217e-07,
|
|
"loss": 0.0056,
|
|
"num_tokens": 25375267.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5092765688896179,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 265,
|
|
"step_time": 298.0915329474956
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3938.0,
|
|
"completions/mean_length": 1631.65625,
|
|
"completions/mean_terminated_length": 1062.9615478515625,
|
|
"completions/min_length": 25.0,
|
|
"completions/min_terminated_length": 25.0,
|
|
"entropy": 5.047824844717979,
|
|
"epoch": 0.15079365079365079,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1591796875,
|
|
"kl": 0.001154161127487896,
|
|
"learning_rate": 8.497732426303855e-07,
|
|
"loss": 0.0211,
|
|
"num_tokens": 25490301.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 266,
|
|
"step_time": 294.235875419341
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3978.0,
|
|
"completions/mean_length": 1112.96875,
|
|
"completions/mean_terminated_length": 804.3793334960938,
|
|
"completions/min_length": 35.0,
|
|
"completions/min_terminated_length": 35.0,
|
|
"entropy": 3.4297971054911613,
|
|
"epoch": 0.15136054421768708,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.416015625,
|
|
"kl": 0.001714525820716517,
|
|
"learning_rate": 8.492063492063492e-07,
|
|
"loss": 0.0191,
|
|
"num_tokens": 25573715.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7175520658493042,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 267,
|
|
"step_time": 300.0728285070509
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3800.0,
|
|
"completions/mean_length": 1327.265625,
|
|
"completions/mean_terminated_length": 752.6226806640625,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 2.689064610749483,
|
|
"epoch": 0.15192743764172337,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1953125,
|
|
"kl": 0.0015337007698690286,
|
|
"learning_rate": 8.486394557823128e-07,
|
|
"loss": 0.0092,
|
|
"num_tokens": 25673220.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 268,
|
|
"step_time": 310.35281988978386
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2822.0,
|
|
"completions/mean_length": 844.140625,
|
|
"completions/mean_terminated_length": 568.559326171875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 2.2734266445040703,
|
|
"epoch": 0.15249433106575963,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.396484375,
|
|
"kl": 0.0019259779910498764,
|
|
"learning_rate": 8.480725623582766e-07,
|
|
"loss": -0.0005,
|
|
"num_tokens": 25740221.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 269,
|
|
"step_time": 308.26017513033
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2713.0,
|
|
"completions/mean_length": 999.296875,
|
|
"completions/mean_terminated_length": 619.0,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"entropy": 2.6519531086087227,
|
|
"epoch": 0.15306122448979592,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.0014964106158004142,
|
|
"learning_rate": 8.475056689342403e-07,
|
|
"loss": 0.0044,
|
|
"num_tokens": 25816448.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.528690755367279,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 270,
|
|
"step_time": 305.5210933703929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3468.0,
|
|
"completions/mean_length": 1258.546875,
|
|
"completions/mean_terminated_length": 910.0877075195312,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.859191494062543,
|
|
"epoch": 0.1536281179138322,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.10693359375,
|
|
"kl": 0.0013441895043797558,
|
|
"learning_rate": 8.469387755102041e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 25908467.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 271,
|
|
"step_time": 296.35443599242717
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4000.0,
|
|
"completions/mean_length": 1257.25,
|
|
"completions/mean_terminated_length": 963.586181640625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.312869865447283,
|
|
"epoch": 0.15419501133786848,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.275390625,
|
|
"kl": 0.00116521892050514,
|
|
"learning_rate": 8.463718820861677e-07,
|
|
"loss": 0.0534,
|
|
"num_tokens": 26000899.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 272,
|
|
"step_time": 293.82065302692354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4063.0,
|
|
"completions/mean_length": 1249.640625,
|
|
"completions/mean_terminated_length": 1008.4237060546875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.00705923140049,
|
|
"epoch": 0.15476190476190477,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.0010208657022303669,
|
|
"learning_rate": 8.458049886621315e-07,
|
|
"loss": -0.0283,
|
|
"num_tokens": 26092732.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 273,
|
|
"step_time": 300.6625357680023
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3971.0,
|
|
"completions/mean_length": 1185.390625,
|
|
"completions/mean_terminated_length": 769.5892944335938,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.417567305266857,
|
|
"epoch": 0.15532879818594103,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.001261524190340424,
|
|
"learning_rate": 8.452380952380952e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 26179717.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 274,
|
|
"step_time": 298.452397396788
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3687.0,
|
|
"completions/mean_length": 1130.609375,
|
|
"completions/mean_terminated_length": 706.982177734375,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 2.985208410769701,
|
|
"epoch": 0.15589569160997732,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.201171875,
|
|
"kl": 0.001736221991450293,
|
|
"learning_rate": 8.44671201814059e-07,
|
|
"loss": -0.0012,
|
|
"num_tokens": 26262564.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6782841682434082,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 275,
|
|
"step_time": 294.14131032209843
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4013.0,
|
|
"completions/mean_length": 1411.953125,
|
|
"completions/mean_terminated_length": 914.9074096679688,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 3.070179019123316,
|
|
"epoch": 0.1564625850340136,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.189453125,
|
|
"kl": 0.0013637927659146953,
|
|
"learning_rate": 8.441043083900226e-07,
|
|
"loss": -0.0005,
|
|
"num_tokens": 26365065.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.39708447456359863,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 276,
|
|
"step_time": 296.15611359570175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4049.0,
|
|
"completions/mean_length": 1408.421875,
|
|
"completions/mean_terminated_length": 1078.368408203125,
|
|
"completions/min_length": 155.0,
|
|
"completions/min_terminated_length": 155.0,
|
|
"entropy": 4.443922260776162,
|
|
"epoch": 0.15702947845804988,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.2333984375,
|
|
"kl": 0.001101307847420685,
|
|
"learning_rate": 8.435374149659864e-07,
|
|
"loss": 0.0635,
|
|
"num_tokens": 26467676.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6664806604385376,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 277,
|
|
"step_time": 296.64144099131227
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3967.0,
|
|
"completions/mean_length": 1329.296875,
|
|
"completions/mean_terminated_length": 1094.83056640625,
|
|
"completions/min_length": 218.0,
|
|
"completions/min_terminated_length": 218.0,
|
|
"entropy": 3.343450289219618,
|
|
"epoch": 0.15759637188208617,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.193359375,
|
|
"kl": 0.0009303500173700741,
|
|
"learning_rate": 8.4297052154195e-07,
|
|
"loss": 0.0212,
|
|
"num_tokens": 26565863.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.41539984941482544,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 278,
|
|
"step_time": 306.69521214906126
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3969.0,
|
|
"completions/mean_length": 1104.734375,
|
|
"completions/mean_terminated_length": 795.2930908203125,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 2.88244404271245,
|
|
"epoch": 0.15816326530612246,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.291015625,
|
|
"kl": 0.0014865316625218838,
|
|
"learning_rate": 8.424036281179138e-07,
|
|
"loss": -0.0034,
|
|
"num_tokens": 26647478.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 279,
|
|
"step_time": 296.1632002191618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3919.0,
|
|
"completions/mean_length": 1359.15625,
|
|
"completions/mean_terminated_length": 968.1785888671875,
|
|
"completions/min_length": 25.0,
|
|
"completions/min_terminated_length": 25.0,
|
|
"entropy": 4.191066320985556,
|
|
"epoch": 0.15873015873015872,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2314453125,
|
|
"kl": 0.0012526897280622507,
|
|
"learning_rate": 8.418367346938775e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 26745072.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 280,
|
|
"step_time": 291.89493420906365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3779.0,
|
|
"completions/mean_length": 1178.59375,
|
|
"completions/mean_terminated_length": 931.35595703125,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.5426559150218964,
|
|
"epoch": 0.159297052154195,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.0011631767283688532,
|
|
"learning_rate": 8.412698412698413e-07,
|
|
"loss": -0.1123,
|
|
"num_tokens": 26832014.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.46922925114631653,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 281,
|
|
"step_time": 300.498527049087
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3771.0,
|
|
"completions/mean_length": 1375.5625,
|
|
"completions/mean_terminated_length": 986.9285888671875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.4970968775451183,
|
|
"epoch": 0.1598639455782313,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2275390625,
|
|
"kl": 0.0015308281963370973,
|
|
"learning_rate": 8.40702947845805e-07,
|
|
"loss": 0.033,
|
|
"num_tokens": 26931434.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 282,
|
|
"step_time": 294.31987982895225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3254.0,
|
|
"completions/mean_length": 1049.328125,
|
|
"completions/mean_terminated_length": 614.0892944335938,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.0270057693123817,
|
|
"epoch": 0.16043083900226757,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1015625,
|
|
"kl": 0.0014629809629695956,
|
|
"learning_rate": 8.401360544217688e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 27012255.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.6131313443183899,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 283,
|
|
"step_time": 303.08907681982964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3777.0,
|
|
"completions/mean_length": 1197.109375,
|
|
"completions/mean_terminated_length": 897.22412109375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.958734929561615,
|
|
"epoch": 0.16099773242630386,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.30078125,
|
|
"kl": 0.0010162165599467698,
|
|
"learning_rate": 8.395691609977323e-07,
|
|
"loss": 0.0478,
|
|
"num_tokens": 27099862.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 284,
|
|
"step_time": 296.52208467107266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3984.0,
|
|
"completions/mean_length": 1357.4375,
|
|
"completions/mean_terminated_length": 850.2963256835938,
|
|
"completions/min_length": 224.0,
|
|
"completions/min_terminated_length": 224.0,
|
|
"entropy": 3.6308336928486824,
|
|
"epoch": 0.16156462585034015,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2197265625,
|
|
"kl": 0.0008931458942242898,
|
|
"learning_rate": 8.39002267573696e-07,
|
|
"loss": 0.0107,
|
|
"num_tokens": 27198162.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 285,
|
|
"step_time": 299.7050953200087
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3696.0,
|
|
"completions/mean_length": 1512.46875,
|
|
"completions/mean_terminated_length": 1089.7091064453125,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"entropy": 4.266504943370819,
|
|
"epoch": 0.1621315192743764,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.125,
|
|
"kl": 0.0014134251723589841,
|
|
"learning_rate": 8.384353741496598e-07,
|
|
"loss": 0.0412,
|
|
"num_tokens": 27306912.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 286,
|
|
"step_time": 298.54193136747926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3498.0,
|
|
"completions/mean_length": 1383.46875,
|
|
"completions/mean_terminated_length": 995.96435546875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.3081691190600395,
|
|
"epoch": 0.1626984126984127,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.177734375,
|
|
"kl": 0.0010290788322890876,
|
|
"learning_rate": 8.378684807256235e-07,
|
|
"loss": 0.0324,
|
|
"num_tokens": 27407398.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.41277948021888733,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 287,
|
|
"step_time": 297.7804460711777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3605.0,
|
|
"completions/mean_length": 1589.84375,
|
|
"completions/mean_terminated_length": 888.1199951171875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.3641773611307144,
|
|
"epoch": 0.16326530612244897,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.001085398758732481,
|
|
"learning_rate": 8.373015873015873e-07,
|
|
"loss": -0.0172,
|
|
"num_tokens": 27521996.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 288,
|
|
"step_time": 298.2156117670238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3755.0,
|
|
"completions/mean_length": 1014.3125,
|
|
"completions/mean_terminated_length": 695.5172119140625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.941107463091612,
|
|
"epoch": 0.16383219954648526,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.36328125,
|
|
"kl": 0.0012372971614240669,
|
|
"learning_rate": 8.36734693877551e-07,
|
|
"loss": 0.0165,
|
|
"num_tokens": 27598744.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 289,
|
|
"step_time": 294.38263658620417
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3727.0,
|
|
"completions/mean_length": 1210.796875,
|
|
"completions/mean_terminated_length": 798.6250610351562,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.147449314594269,
|
|
"epoch": 0.16439909297052155,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2255859375,
|
|
"kl": 0.0011653340752673103,
|
|
"learning_rate": 8.361678004535147e-07,
|
|
"loss": -0.0157,
|
|
"num_tokens": 27687795.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 290,
|
|
"step_time": 296.3157152188942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4025.0,
|
|
"completions/mean_length": 1743.390625,
|
|
"completions/mean_terminated_length": 1143.7059326171875,
|
|
"completions/min_length": 243.0,
|
|
"completions/min_terminated_length": 243.0,
|
|
"entropy": 3.946227416396141,
|
|
"epoch": 0.1649659863945578,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1328125,
|
|
"kl": 0.0010434207342768786,
|
|
"learning_rate": 8.356009070294783e-07,
|
|
"loss": 0.028,
|
|
"num_tokens": 27812332.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 291,
|
|
"step_time": 309.8255847459659
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3957.0,
|
|
"completions/mean_length": 1133.796875,
|
|
"completions/mean_terminated_length": 827.362060546875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.052217453718185,
|
|
"epoch": 0.1655328798185941,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1083984375,
|
|
"kl": 0.0013799356638628524,
|
|
"learning_rate": 8.350340136054421e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 27900503.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.33174845576286316,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 292,
|
|
"step_time": 313.85175424348563
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2972.0,
|
|
"completions/mean_length": 1030.1875,
|
|
"completions/mean_terminated_length": 713.0344848632812,
|
|
"completions/min_length": 38.0,
|
|
"completions/min_terminated_length": 38.0,
|
|
"entropy": 3.4501704461872578,
|
|
"epoch": 0.1660997732426304,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2080078125,
|
|
"kl": 0.0015028614325274248,
|
|
"learning_rate": 8.344671201814058e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 27978755.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 293,
|
|
"step_time": 302.06553180422634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3667.0,
|
|
"completions/mean_length": 1097.328125,
|
|
"completions/mean_terminated_length": 949.8524169921875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.379862379282713,
|
|
"epoch": 0.16666666666666666,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1943359375,
|
|
"kl": 0.002225910628112615,
|
|
"learning_rate": 8.339002267573696e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 28063072.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5554563403129578,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 294,
|
|
"step_time": 305.31014814879745
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3855.0,
|
|
"completions/mean_length": 1184.140625,
|
|
"completions/mean_terminated_length": 768.1607666015625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.5274267680943012,
|
|
"epoch": 0.16723356009070295,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.41796875,
|
|
"kl": 0.0014518765474349493,
|
|
"learning_rate": 8.333333333333333e-07,
|
|
"loss": -0.1013,
|
|
"num_tokens": 28150961.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 295,
|
|
"step_time": 300.1005868501961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3222.0,
|
|
"completions/mean_length": 1218.96875,
|
|
"completions/mean_terminated_length": 865.6491088867188,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.963806878775358,
|
|
"epoch": 0.16780045351473924,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.0015549707040918292,
|
|
"learning_rate": 8.327664399092971e-07,
|
|
"loss": -0.092,
|
|
"num_tokens": 28240447.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5890451669692993,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 296,
|
|
"step_time": 296.3705997876823
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3883.0,
|
|
"completions/mean_length": 1019.421875,
|
|
"completions/mean_terminated_length": 814.3167114257812,
|
|
"completions/min_length": 121.0,
|
|
"completions/min_terminated_length": 121.0,
|
|
"entropy": 3.574819404631853,
|
|
"epoch": 0.1683673469387755,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.0015025965749373427,
|
|
"learning_rate": 8.321995464852607e-07,
|
|
"loss": 0.1117,
|
|
"num_tokens": 28317514.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 297,
|
|
"step_time": 294.27986282110214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3922.0,
|
|
"completions/mean_length": 1468.890625,
|
|
"completions/mean_terminated_length": 923.6415405273438,
|
|
"completions/min_length": 94.0,
|
|
"completions/min_terminated_length": 94.0,
|
|
"entropy": 3.008502796292305,
|
|
"epoch": 0.1689342403628118,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.166015625,
|
|
"kl": 0.001358457573587657,
|
|
"learning_rate": 8.316326530612245e-07,
|
|
"loss": 0.0075,
|
|
"num_tokens": 28423035.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 298,
|
|
"step_time": 293.9276619926095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3777.0,
|
|
"completions/mean_length": 1020.203125,
|
|
"completions/mean_terminated_length": 642.4736938476562,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.857605691999197,
|
|
"epoch": 0.16950113378684808,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2109375,
|
|
"kl": 0.0017286588372371625,
|
|
"learning_rate": 8.310657596371881e-07,
|
|
"loss": 0.0104,
|
|
"num_tokens": 28500880.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 299,
|
|
"step_time": 302.13770243152976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3957.0,
|
|
"completions/mean_length": 1521.671875,
|
|
"completions/mean_terminated_length": 1100.4180908203125,
|
|
"completions/min_length": 158.0,
|
|
"completions/min_terminated_length": 158.0,
|
|
"entropy": 4.238505080342293,
|
|
"epoch": 0.17006802721088435,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.140625,
|
|
"kl": 0.001234137591382023,
|
|
"learning_rate": 8.304988662131519e-07,
|
|
"loss": 0.0313,
|
|
"num_tokens": 28611323.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 300,
|
|
"step_time": 302.78664544876665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3890.0,
|
|
"completions/mean_length": 1194.28125,
|
|
"completions/mean_terminated_length": 779.7500610351562,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"entropy": 2.7570454962551594,
|
|
"epoch": 0.17063492063492064,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.361328125,
|
|
"kl": 0.0012360909440758405,
|
|
"learning_rate": 8.299319727891156e-07,
|
|
"loss": -0.0846,
|
|
"num_tokens": 28700133.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4787135720252991,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 301,
|
|
"step_time": 297.30019121896476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3754.0,
|
|
"completions/mean_length": 1068.0625,
|
|
"completions/mean_terminated_length": 754.8275756835938,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.0655543729662895,
|
|
"epoch": 0.1712018140589569,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.0014381433647940867,
|
|
"learning_rate": 8.293650793650794e-07,
|
|
"loss": 0.0318,
|
|
"num_tokens": 28780617.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5719553828239441,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 302,
|
|
"step_time": 294.7732902197167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3913.0,
|
|
"completions/mean_length": 1324.046875,
|
|
"completions/mean_terminated_length": 748.73583984375,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 2.493778932839632,
|
|
"epoch": 0.1717687074829932,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.0013097982155159116,
|
|
"learning_rate": 8.28798185941043e-07,
|
|
"loss": 0.0305,
|
|
"num_tokens": 28876980.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 303,
|
|
"step_time": 295.6098387800157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4044.0,
|
|
"completions/mean_length": 1439.65625,
|
|
"completions/mean_terminated_length": 1060.1785888671875,
|
|
"completions/min_length": 96.0,
|
|
"completions/min_terminated_length": 96.0,
|
|
"entropy": 5.044021558016539,
|
|
"epoch": 0.17233560090702948,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.228515625,
|
|
"kl": 0.0010676935780793428,
|
|
"learning_rate": 8.282312925170068e-07,
|
|
"loss": 0.0166,
|
|
"num_tokens": 28978598.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 304,
|
|
"step_time": 290.7202107226476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3698.0,
|
|
"completions/mean_length": 1337.046875,
|
|
"completions/mean_terminated_length": 764.4339599609375,
|
|
"completions/min_length": 43.0,
|
|
"completions/min_terminated_length": 43.0,
|
|
"entropy": 3.115324232727289,
|
|
"epoch": 0.17290249433106575,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.248046875,
|
|
"kl": 0.00165786925208522,
|
|
"learning_rate": 8.276643990929705e-07,
|
|
"loss": 0.0409,
|
|
"num_tokens": 29076329.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 305,
|
|
"step_time": 305.97279978822917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3989.0,
|
|
"completions/mean_length": 1449.640625,
|
|
"completions/mean_terminated_length": 959.5740966796875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.806913925334811,
|
|
"epoch": 0.17346938775510204,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.140625,
|
|
"kl": 0.0014475634743575938,
|
|
"learning_rate": 8.270975056689343e-07,
|
|
"loss": -0.0037,
|
|
"num_tokens": 29182658.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.7125226259231567,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 306,
|
|
"step_time": 304.92379070911556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3885.0,
|
|
"completions/mean_length": 1247.125,
|
|
"completions/mean_terminated_length": 1057.2000732421875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.925425849854946,
|
|
"epoch": 0.17403628117913833,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.0012668101971939905,
|
|
"learning_rate": 8.26530612244898e-07,
|
|
"loss": -0.0205,
|
|
"num_tokens": 29274002.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 307,
|
|
"step_time": 301.54330573696643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3863.0,
|
|
"completions/mean_length": 1325.75,
|
|
"completions/mean_terminated_length": 1039.17236328125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.7136062681674957,
|
|
"epoch": 0.1746031746031746,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1337890625,
|
|
"kl": 0.0014722158321092138,
|
|
"learning_rate": 8.259637188208616e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 29376322.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5866720080375671,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 308,
|
|
"step_time": 326.7536249374971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4049.0,
|
|
"completions/mean_length": 1406.890625,
|
|
"completions/mean_terminated_length": 908.9074096679688,
|
|
"completions/min_length": 153.0,
|
|
"completions/min_terminated_length": 153.0,
|
|
"entropy": 3.7738971263170242,
|
|
"epoch": 0.17517006802721088,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2294921875,
|
|
"kl": 0.0013544335824917653,
|
|
"learning_rate": 8.253968253968253e-07,
|
|
"loss": 0.0256,
|
|
"num_tokens": 29478883.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.46691107749938965,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 309,
|
|
"step_time": 296.07146343030035
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3803.0,
|
|
"completions/mean_length": 1261.296875,
|
|
"completions/mean_terminated_length": 1072.3167724609375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.0862605310976505,
|
|
"epoch": 0.17573696145124718,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.359375,
|
|
"kl": 0.0015789931640028954,
|
|
"learning_rate": 8.24829931972789e-07,
|
|
"loss": 0.0932,
|
|
"num_tokens": 29571286.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 310,
|
|
"step_time": 298.1583162872121
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4042.0,
|
|
"completions/mean_length": 920.015625,
|
|
"completions/mean_terminated_length": 817.5645141601562,
|
|
"completions/min_length": 26.0,
|
|
"completions/min_terminated_length": 26.0,
|
|
"entropy": 3.515986368060112,
|
|
"epoch": 0.17630385487528344,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.0011895883180841338,
|
|
"learning_rate": 8.242630385487528e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 29641503.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 311,
|
|
"step_time": 295.0998282721266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3785.0,
|
|
"completions/mean_length": 1333.203125,
|
|
"completions/mean_terminated_length": 821.5740966796875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.287606105208397,
|
|
"epoch": 0.17687074829931973,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.32421875,
|
|
"kl": 0.001888270318886498,
|
|
"learning_rate": 8.236961451247165e-07,
|
|
"loss": -0.0212,
|
|
"num_tokens": 29738348.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.69987952709198,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 312,
|
|
"step_time": 297.20515647530556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3803.0,
|
|
"completions/mean_length": 1015.25,
|
|
"completions/mean_terminated_length": 754.1694946289062,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.613906405866146,
|
|
"epoch": 0.17743764172335602,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.376953125,
|
|
"kl": 0.001490443704824429,
|
|
"learning_rate": 8.231292517006803e-07,
|
|
"loss": 0.0632,
|
|
"num_tokens": 29814492.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 313,
|
|
"step_time": 292.44761195778847
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3079.0,
|
|
"completions/mean_length": 1275.890625,
|
|
"completions/mean_terminated_length": 753.6481323242188,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.8377242982387543,
|
|
"epoch": 0.17800453514739228,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.23828125,
|
|
"kl": 0.0013854403405275661,
|
|
"learning_rate": 8.22562358276644e-07,
|
|
"loss": 0.0308,
|
|
"num_tokens": 29907221.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 314,
|
|
"step_time": 291.80689421482384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3869.0,
|
|
"completions/mean_length": 1183.765625,
|
|
"completions/mean_terminated_length": 826.122802734375,
|
|
"completions/min_length": 36.0,
|
|
"completions/min_terminated_length": 36.0,
|
|
"entropy": 4.167344301939011,
|
|
"epoch": 0.17857142857142858,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.0011127866673632525,
|
|
"learning_rate": 8.219954648526076e-07,
|
|
"loss": 0.077,
|
|
"num_tokens": 29995166.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 315,
|
|
"step_time": 305.0239759385586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3820.0,
|
|
"completions/mean_length": 1142.765625,
|
|
"completions/mean_terminated_length": 780.0877075195312,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 1.9886627234518528,
|
|
"epoch": 0.17913832199546487,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.240234375,
|
|
"kl": 0.001561849749123212,
|
|
"learning_rate": 8.214285714285713e-07,
|
|
"loss": -0.0347,
|
|
"num_tokens": 30081383.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 316,
|
|
"step_time": 302.644008141011
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3916.0,
|
|
"completions/mean_length": 1109.390625,
|
|
"completions/mean_terminated_length": 962.5081176757812,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 4.265066012740135,
|
|
"epoch": 0.17970521541950113,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.35546875,
|
|
"kl": 0.0012646747200051323,
|
|
"learning_rate": 8.208616780045351e-07,
|
|
"loss": 0.0464,
|
|
"num_tokens": 30164376.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 317,
|
|
"step_time": 296.1302981870249
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3752.0,
|
|
"completions/mean_length": 800.109375,
|
|
"completions/mean_terminated_length": 693.790283203125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.901810046285391,
|
|
"epoch": 0.18027210884353742,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.0015241189685184509,
|
|
"learning_rate": 8.202947845804988e-07,
|
|
"loss": -0.1508,
|
|
"num_tokens": 30227879.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 318,
|
|
"step_time": 300.9886961122975
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3451.0,
|
|
"completions/mean_length": 1135.765625,
|
|
"completions/mean_terminated_length": 772.2280883789062,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"entropy": 3.722517892718315,
|
|
"epoch": 0.18083900226757368,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2109375,
|
|
"kl": 0.0013234901507530594,
|
|
"learning_rate": 8.197278911564626e-07,
|
|
"loss": 0.0294,
|
|
"num_tokens": 30311352.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 319,
|
|
"step_time": 291.4153741868213
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4090.0,
|
|
"completions/mean_length": 1477.09375,
|
|
"completions/mean_terminated_length": 992.1111450195312,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.208458241075277,
|
|
"epoch": 0.18140589569160998,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.11181640625,
|
|
"kl": 0.0012379130166664254,
|
|
"learning_rate": 8.191609977324263e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 30417158.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 320,
|
|
"step_time": 293.3395473388955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3407.0,
|
|
"completions/mean_length": 969.90625,
|
|
"completions/mean_terminated_length": 704.9830322265625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.662312377244234,
|
|
"epoch": 0.18197278911564627,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.001847629977419274,
|
|
"learning_rate": 8.1859410430839e-07,
|
|
"loss": 0.0102,
|
|
"num_tokens": 30491000.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6993034482002258,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 321,
|
|
"step_time": 296.2531007146463
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4051.0,
|
|
"completions/mean_length": 795.203125,
|
|
"completions/mean_terminated_length": 688.7257690429688,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8032184205949306,
|
|
"epoch": 0.18253968253968253,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.0014939455886633368,
|
|
"learning_rate": 8.180272108843537e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 30553837.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 322,
|
|
"step_time": 300.56049086526036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2537.0,
|
|
"completions/mean_length": 988.140625,
|
|
"completions/mean_terminated_length": 666.637939453125,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 1.7907675132155418,
|
|
"epoch": 0.18310657596371882,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.310546875,
|
|
"kl": 0.00149022775440244,
|
|
"learning_rate": 8.174603174603174e-07,
|
|
"loss": 0.0241,
|
|
"num_tokens": 30627958.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 323,
|
|
"step_time": 292.3509076265618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4004.0,
|
|
"completions/mean_length": 1345.296875,
|
|
"completions/mean_terminated_length": 952.33935546875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.6916390731930733,
|
|
"epoch": 0.1836734693877551,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2275390625,
|
|
"kl": 0.001267036413992173,
|
|
"learning_rate": 8.168934240362811e-07,
|
|
"loss": 0.0096,
|
|
"num_tokens": 30724769.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 324,
|
|
"step_time": 293.23362885788083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3688.0,
|
|
"completions/mean_length": 1114.75,
|
|
"completions/mean_terminated_length": 806.3448486328125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.472703419625759,
|
|
"epoch": 0.18424036281179137,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.33984375,
|
|
"kl": 0.0011231746357225347,
|
|
"learning_rate": 8.163265306122449e-07,
|
|
"loss": 0.0674,
|
|
"num_tokens": 30808553.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 325,
|
|
"step_time": 299.0592782208696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4014.0,
|
|
"completions/mean_length": 1138.828125,
|
|
"completions/mean_terminated_length": 775.6666870117188,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.44533871114254,
|
|
"epoch": 0.18480725623582767,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.001377893022436183,
|
|
"learning_rate": 8.157596371882086e-07,
|
|
"loss": -0.0992,
|
|
"num_tokens": 30893550.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 326,
|
|
"step_time": 296.04104791674763
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4055.0,
|
|
"completions/mean_length": 1251.875,
|
|
"completions/mean_terminated_length": 1010.8474731445312,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.31509043648839,
|
|
"epoch": 0.18537414965986396,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2353515625,
|
|
"kl": 0.0012605721276486292,
|
|
"learning_rate": 8.151927437641724e-07,
|
|
"loss": 0.0133,
|
|
"num_tokens": 30986430.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 327,
|
|
"step_time": 305.77011601254344
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3758.0,
|
|
"completions/mean_length": 1622.890625,
|
|
"completions/mean_terminated_length": 1319.1754150390625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.529617294669151,
|
|
"epoch": 0.18594104308390022,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1474609375,
|
|
"kl": 0.0010493327517906437,
|
|
"learning_rate": 8.14625850340136e-07,
|
|
"loss": 0.0473,
|
|
"num_tokens": 31101519.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 328,
|
|
"step_time": 292.69737570080906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4076.0,
|
|
"completions/mean_length": 1463.6875,
|
|
"completions/mean_terminated_length": 976.2222290039062,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 3.6420319005846977,
|
|
"epoch": 0.1865079365079365,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2001953125,
|
|
"kl": 0.001224763143909513,
|
|
"learning_rate": 8.140589569160997e-07,
|
|
"loss": 0.0244,
|
|
"num_tokens": 31207643.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 329,
|
|
"step_time": 296.2488160226494
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4065.0,
|
|
"completions/mean_length": 1161.46875,
|
|
"completions/mean_terminated_length": 857.8965454101562,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.393191885203123,
|
|
"epoch": 0.1870748299319728,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.0013154964290151838,
|
|
"learning_rate": 8.134920634920635e-07,
|
|
"loss": 0.0648,
|
|
"num_tokens": 31293169.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 330,
|
|
"step_time": 293.66058210935444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3771.0,
|
|
"completions/mean_length": 1535.578125,
|
|
"completions/mean_terminated_length": 1004.1697998046875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 5.231051322072744,
|
|
"epoch": 0.18764172335600907,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1689453125,
|
|
"kl": 0.001104443465010263,
|
|
"learning_rate": 8.129251700680271e-07,
|
|
"loss": 0.0174,
|
|
"num_tokens": 31402286.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 331,
|
|
"step_time": 292.3300448982045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3969.0,
|
|
"completions/mean_length": 1272.453125,
|
|
"completions/mean_terminated_length": 810.4181518554688,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7793682869523764,
|
|
"epoch": 0.18820861678004536,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.04931640625,
|
|
"kl": 0.0014106326598266605,
|
|
"learning_rate": 8.123582766439909e-07,
|
|
"loss": -0.0106,
|
|
"num_tokens": 31498211.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 332,
|
|
"step_time": 307.6453470662236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3822.0,
|
|
"completions/mean_length": 1324.375,
|
|
"completions/mean_terminated_length": 928.4285888671875,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 2.3627333492040634,
|
|
"epoch": 0.18877551020408162,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.001209826299600536,
|
|
"learning_rate": 8.117913832199546e-07,
|
|
"loss": -0.0133,
|
|
"num_tokens": 31594515.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 333,
|
|
"step_time": 295.49152721557766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3381.0,
|
|
"completions/mean_length": 983.875,
|
|
"completions/mean_terminated_length": 776.4000244140625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.2466306276619434,
|
|
"epoch": 0.1893424036281179,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.25390625,
|
|
"kl": 0.0015662444275221787,
|
|
"learning_rate": 8.112244897959183e-07,
|
|
"loss": -0.0071,
|
|
"num_tokens": 31668547.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 334,
|
|
"step_time": 292.95015636552125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3820.0,
|
|
"completions/mean_length": 1548.234375,
|
|
"completions/mean_terminated_length": 960.2885131835938,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.4881539437919855,
|
|
"epoch": 0.1899092970521542,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.140625,
|
|
"kl": 0.0013060569126537303,
|
|
"learning_rate": 8.10657596371882e-07,
|
|
"loss": 0.0143,
|
|
"num_tokens": 31779666.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 335,
|
|
"step_time": 293.80418958794326
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3865.0,
|
|
"completions/mean_length": 1332.578125,
|
|
"completions/mean_terminated_length": 880.3817749023438,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.352604014798999,
|
|
"epoch": 0.19047619047619047,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.189453125,
|
|
"kl": 0.00127808340221236,
|
|
"learning_rate": 8.100907029478458e-07,
|
|
"loss": 0.0419,
|
|
"num_tokens": 31875927.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 336,
|
|
"step_time": 293.79132070112973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3297.0,
|
|
"completions/mean_length": 1400.984375,
|
|
"completions/mean_terminated_length": 901.9074096679688,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.137748945504427,
|
|
"epoch": 0.19104308390022676,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.166015625,
|
|
"kl": 0.0011957738133787643,
|
|
"learning_rate": 8.095238095238095e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 31977318.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 337,
|
|
"step_time": 300.70368077233434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3987.0,
|
|
"completions/mean_length": 1183.015625,
|
|
"completions/mean_terminated_length": 766.8750610351562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.303982373327017,
|
|
"epoch": 0.19160997732426305,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.0018087913922499865,
|
|
"learning_rate": 8.089569160997733e-07,
|
|
"loss": 0.0331,
|
|
"num_tokens": 32066511.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 338,
|
|
"step_time": 298.1635686745867
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2989.0,
|
|
"completions/mean_length": 872.140625,
|
|
"completions/mean_terminated_length": 657.2166748046875,
|
|
"completions/min_length": 169.0,
|
|
"completions/min_terminated_length": 169.0,
|
|
"entropy": 2.553455151617527,
|
|
"epoch": 0.1921768707482993,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.0014670600940007716,
|
|
"learning_rate": 8.083900226757369e-07,
|
|
"loss": 0.0204,
|
|
"num_tokens": 32135224.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 339,
|
|
"step_time": 305.7137638395652
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3732.0,
|
|
"completions/mean_length": 1285.421875,
|
|
"completions/mean_terminated_length": 702.0943603515625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.686339009553194,
|
|
"epoch": 0.1927437641723356,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.11181640625,
|
|
"kl": 0.0014709026254422497,
|
|
"learning_rate": 8.078231292517006e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 32229171.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 340,
|
|
"step_time": 294.1925411038101
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3938.0,
|
|
"completions/mean_length": 1565.140625,
|
|
"completions/mean_terminated_length": 920.0196533203125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.788554474711418,
|
|
"epoch": 0.1933106575963719,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1015625,
|
|
"kl": 0.0017092519883590285,
|
|
"learning_rate": 8.072562358276643e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 32343316.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 341,
|
|
"step_time": 300.09620239492506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2418.0,
|
|
"completions/mean_length": 1192.125,
|
|
"completions/mean_terminated_length": 716.9454345703125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 1.80996223539114,
|
|
"epoch": 0.19387755102040816,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2216796875,
|
|
"kl": 0.001654357838560827,
|
|
"learning_rate": 8.066893424036281e-07,
|
|
"loss": 0.0122,
|
|
"num_tokens": 32430700.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 342,
|
|
"step_time": 296.0148100089282
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3730.0,
|
|
"completions/mean_length": 1481.6875,
|
|
"completions/mean_terminated_length": 878.3846435546875,
|
|
"completions/min_length": 99.0,
|
|
"completions/min_terminated_length": 99.0,
|
|
"entropy": 2.1769518852233887,
|
|
"epoch": 0.19444444444444445,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.14453125,
|
|
"kl": 0.00137901808557217,
|
|
"learning_rate": 8.061224489795918e-07,
|
|
"loss": -0.0012,
|
|
"num_tokens": 32537680.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 343,
|
|
"step_time": 296.50169356074184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4093.0,
|
|
"completions/mean_length": 1020.703125,
|
|
"completions/mean_terminated_length": 760.084716796875,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"entropy": 2.139278154820204,
|
|
"epoch": 0.19501133786848074,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.337890625,
|
|
"kl": 0.0013905934702052036,
|
|
"learning_rate": 8.055555555555556e-07,
|
|
"loss": 0.0395,
|
|
"num_tokens": 32614197.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 344,
|
|
"step_time": 295.34965351130813
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3910.0,
|
|
"completions/mean_length": 1866.703125,
|
|
"completions/mean_terminated_length": 1548.232177734375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 6.025232031941414,
|
|
"epoch": 0.195578231292517,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.107421875,
|
|
"kl": 0.0009480748649366433,
|
|
"learning_rate": 8.049886621315193e-07,
|
|
"loss": -0.0161,
|
|
"num_tokens": 32747778.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 345,
|
|
"step_time": 301.9893612070009
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3226.0,
|
|
"completions/mean_length": 1214.71875,
|
|
"completions/mean_terminated_length": 681.1481323242188,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 2.4195906836539507,
|
|
"epoch": 0.1961451247165533,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1416015625,
|
|
"kl": 0.0014571854117093608,
|
|
"learning_rate": 8.04421768707483e-07,
|
|
"loss": 0.0527,
|
|
"num_tokens": 32837936.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 346,
|
|
"step_time": 296.6905417237431
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3731.0,
|
|
"completions/mean_length": 1326.546875,
|
|
"completions/mean_terminated_length": 930.9107666015625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.9649287778884172,
|
|
"epoch": 0.19671201814058956,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.0010013702285505133,
|
|
"learning_rate": 8.038548752834466e-07,
|
|
"loss": 0.0383,
|
|
"num_tokens": 32932835.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 347,
|
|
"step_time": 290.2288642572239
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3716.0,
|
|
"completions/mean_length": 1499.609375,
|
|
"completions/mean_terminated_length": 1018.7963256835938,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.093114998191595,
|
|
"epoch": 0.19727891156462585,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.14453125,
|
|
"kl": 0.0013866425415471895,
|
|
"learning_rate": 8.032879818594103e-07,
|
|
"loss": 0.0199,
|
|
"num_tokens": 33039826.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 348,
|
|
"step_time": 292.7541776187718
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4085.0,
|
|
"completions/mean_length": 1253.203125,
|
|
"completions/mean_terminated_length": 847.0892944335938,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.132527057081461,
|
|
"epoch": 0.19784580498866214,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1337890625,
|
|
"kl": 0.001471931976993801,
|
|
"learning_rate": 8.027210884353741e-07,
|
|
"loss": 0.0275,
|
|
"num_tokens": 33132607.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.33174845576286316,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 349,
|
|
"step_time": 302.33250446710736
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3581.0,
|
|
"completions/mean_length": 1014.09375,
|
|
"completions/mean_terminated_length": 808.6333618164062,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 3.1049369759857655,
|
|
"epoch": 0.1984126984126984,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3203125,
|
|
"kl": 0.0014407995804504026,
|
|
"learning_rate": 8.021541950113378e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 33214365.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 350,
|
|
"step_time": 343.75238325074315
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3666.0,
|
|
"completions/mean_length": 1014.90625,
|
|
"completions/mean_terminated_length": 696.1724243164062,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.3992355689406395,
|
|
"epoch": 0.1989795918367347,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"kl": 0.0015600469305354636,
|
|
"learning_rate": 8.015873015873016e-07,
|
|
"loss": -0.0205,
|
|
"num_tokens": 33291855.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 351,
|
|
"step_time": 309.2162681380287
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3978.0,
|
|
"completions/mean_length": 938.9375,
|
|
"completions/mean_terminated_length": 837.0967407226562,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 2.9943065233528614,
|
|
"epoch": 0.19954648526077098,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2412109375,
|
|
"kl": 0.0011394561715860618,
|
|
"learning_rate": 8.010204081632652e-07,
|
|
"loss": 0.0025,
|
|
"num_tokens": 33362971.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 352,
|
|
"step_time": 294.9366546738893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4009.0,
|
|
"completions/mean_length": 1198.15625,
|
|
"completions/mean_terminated_length": 898.3793334960938,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.235837150365114,
|
|
"epoch": 0.20011337868480725,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.171875,
|
|
"kl": 0.0013127148977218894,
|
|
"learning_rate": 8.00453514739229e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 33451005.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 353,
|
|
"step_time": 300.2821004744619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4045.0,
|
|
"completions/mean_length": 1028.609375,
|
|
"completions/mean_terminated_length": 768.6610107421875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8050526566803455,
|
|
"epoch": 0.20068027210884354,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.224609375,
|
|
"kl": 0.0013078464253339916,
|
|
"learning_rate": 7.998866213151927e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 33528148.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5754139423370361,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 354,
|
|
"step_time": 295.9015074791387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3844.0,
|
|
"completions/mean_length": 1436.625,
|
|
"completions/mean_terminated_length": 822.923095703125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.596452224999666,
|
|
"epoch": 0.20124716553287983,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.142578125,
|
|
"kl": 0.0013235371661721729,
|
|
"learning_rate": 7.993197278911564e-07,
|
|
"loss": 0.0163,
|
|
"num_tokens": 33630372.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 355,
|
|
"step_time": 291.1839153263718
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3704.0,
|
|
"completions/mean_length": 1521.9375,
|
|
"completions/mean_terminated_length": 1045.25927734375,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 3.9435150884091854,
|
|
"epoch": 0.2018140589569161,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1943359375,
|
|
"kl": 0.0012276593606657116,
|
|
"learning_rate": 7.987528344671201e-07,
|
|
"loss": 0.0343,
|
|
"num_tokens": 33743432.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 356,
|
|
"step_time": 320.05601400416344
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3949.0,
|
|
"completions/mean_length": 1252.609375,
|
|
"completions/mean_terminated_length": 958.4655151367188,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 3.4948107935488224,
|
|
"epoch": 0.20238095238095238,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.0015216540377878118,
|
|
"learning_rate": 7.981859410430839e-07,
|
|
"loss": 0.0455,
|
|
"num_tokens": 33833911.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 357,
|
|
"step_time": 293.9998562326655
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3068.0,
|
|
"completions/mean_length": 1035.0,
|
|
"completions/mean_terminated_length": 775.5932006835938,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 2.5392488501966,
|
|
"epoch": 0.20294784580498867,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.00149408013021457,
|
|
"learning_rate": 7.976190476190476e-07,
|
|
"loss": -0.0005,
|
|
"num_tokens": 33912631.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 358,
|
|
"step_time": 306.4622711185366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3355.0,
|
|
"completions/mean_length": 815.796875,
|
|
"completions/mean_terminated_length": 709.9838256835938,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.474315457046032,
|
|
"epoch": 0.20351473922902494,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.474609375,
|
|
"kl": 0.0015659075288567692,
|
|
"learning_rate": 7.970521541950113e-07,
|
|
"loss": 0.1007,
|
|
"num_tokens": 33976834.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 359,
|
|
"step_time": 299.45716447010636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4089.0,
|
|
"completions/mean_length": 1094.6875,
|
|
"completions/mean_terminated_length": 894.6000366210938,
|
|
"completions/min_length": 50.0,
|
|
"completions/min_terminated_length": 50.0,
|
|
"entropy": 3.4159761611372232,
|
|
"epoch": 0.20408163265306123,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.0012001038285234245,
|
|
"learning_rate": 7.96485260770975e-07,
|
|
"loss": 0.0838,
|
|
"num_tokens": 34057582.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 360,
|
|
"step_time": 295.3705814424902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3910.0,
|
|
"completions/mean_length": 1578.796875,
|
|
"completions/mean_terminated_length": 1112.648193359375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.302098356187344,
|
|
"epoch": 0.2046485260770975,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.09814453125,
|
|
"kl": 0.0013481216483341996,
|
|
"learning_rate": 7.959183673469388e-07,
|
|
"loss": 0.01,
|
|
"num_tokens": 34170953.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 361,
|
|
"step_time": 299.9199784323573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4005.0,
|
|
"completions/mean_length": 1218.203125,
|
|
"completions/mean_terminated_length": 920.5,
|
|
"completions/min_length": 222.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"entropy": 3.2134172283113003,
|
|
"epoch": 0.20521541950113378,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.0012985770554223564,
|
|
"learning_rate": 7.953514739229025e-07,
|
|
"loss": 0.0405,
|
|
"num_tokens": 34260014.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 362,
|
|
"step_time": 292.89271545317024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4052.0,
|
|
"completions/mean_length": 1267.3125,
|
|
"completions/mean_terminated_length": 804.4363403320312,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.3208404891192913,
|
|
"epoch": 0.20578231292517007,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.0013819267478538677,
|
|
"learning_rate": 7.947845804988662e-07,
|
|
"loss": 0.0248,
|
|
"num_tokens": 34352338.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 363,
|
|
"step_time": 294.02128049358726
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3690.0,
|
|
"completions/mean_length": 1085.859375,
|
|
"completions/mean_terminated_length": 716.1929931640625,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.4948936328291893,
|
|
"epoch": 0.20634920634920634,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.001578690076712519,
|
|
"learning_rate": 7.942176870748299e-07,
|
|
"loss": 0.0191,
|
|
"num_tokens": 34435649.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5883605480194092,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 364,
|
|
"step_time": 304.2129575004801
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4053.0,
|
|
"completions/mean_length": 1091.1875,
|
|
"completions/mean_terminated_length": 780.3448486328125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.3134211003780365,
|
|
"epoch": 0.20691609977324263,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.001404934359015897,
|
|
"learning_rate": 7.936507936507936e-07,
|
|
"loss": 0.0356,
|
|
"num_tokens": 34515757.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6662480235099792,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 365,
|
|
"step_time": 292.800573060289
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3431.0,
|
|
"completions/mean_length": 963.0,
|
|
"completions/mean_terminated_length": 754.1333618164062,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.9491483457386494,
|
|
"epoch": 0.20748299319727892,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.412109375,
|
|
"kl": 0.0019139847026963253,
|
|
"learning_rate": 7.930839002267573e-07,
|
|
"loss": 0.0073,
|
|
"num_tokens": 34591013.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 366,
|
|
"step_time": 306.374175994657
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3739.0,
|
|
"completions/mean_length": 1556.3125,
|
|
"completions/mean_terminated_length": 970.2308349609375,
|
|
"completions/min_length": 114.0,
|
|
"completions/min_terminated_length": 114.0,
|
|
"entropy": 3.646450661122799,
|
|
"epoch": 0.20804988662131518,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1650390625,
|
|
"kl": 0.001971228062757291,
|
|
"learning_rate": 7.92517006802721e-07,
|
|
"loss": -0.0172,
|
|
"num_tokens": 34702361.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 367,
|
|
"step_time": 296.2610024763271
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3948.0,
|
|
"completions/mean_length": 1494.671875,
|
|
"completions/mean_terminated_length": 831.5882568359375,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 4.0113005340099335,
|
|
"epoch": 0.20861678004535147,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.001619795017177239,
|
|
"learning_rate": 7.919501133786848e-07,
|
|
"loss": 0.0211,
|
|
"num_tokens": 34810428.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 368,
|
|
"step_time": 297.6598177989945
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4086.0,
|
|
"completions/mean_length": 1478.078125,
|
|
"completions/mean_terminated_length": 1104.08935546875,
|
|
"completions/min_length": 31.0,
|
|
"completions/min_terminated_length": 31.0,
|
|
"entropy": 3.442749384790659,
|
|
"epoch": 0.20918367346938777,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1806640625,
|
|
"kl": 0.0011501683402457274,
|
|
"learning_rate": 7.913832199546485e-07,
|
|
"loss": 0.0666,
|
|
"num_tokens": 34916737.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 369,
|
|
"step_time": 296.4353799261153
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3747.0,
|
|
"completions/mean_length": 1408.828125,
|
|
"completions/mean_terminated_length": 969.1090698242188,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.96785619109869,
|
|
"epoch": 0.20975056689342403,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.0015126020189200062,
|
|
"learning_rate": 7.908163265306123e-07,
|
|
"loss": 0.0936,
|
|
"num_tokens": 35019102.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.8090203404426575,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 370,
|
|
"step_time": 295.2090236181393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3804.0,
|
|
"completions/mean_length": 830.734375,
|
|
"completions/mean_terminated_length": 613.050048828125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.1338421292603016,
|
|
"epoch": 0.21031746031746032,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.0019984587888757233,
|
|
"learning_rate": 7.902494331065758e-07,
|
|
"loss": 0.0544,
|
|
"num_tokens": 35084013.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 371,
|
|
"step_time": 294.6177222598344
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3749.0,
|
|
"completions/mean_length": 1224.578125,
|
|
"completions/mean_terminated_length": 754.7090454101562,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.497921671718359,
|
|
"epoch": 0.2108843537414966,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.25390625,
|
|
"kl": 0.0022498509297292912,
|
|
"learning_rate": 7.896825396825396e-07,
|
|
"loss": 0.0223,
|
|
"num_tokens": 35172866.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5261042714118958,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 372,
|
|
"step_time": 292.2975083729252
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3942.0,
|
|
"completions/mean_length": 1154.328125,
|
|
"completions/mean_terminated_length": 734.0892944335938,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 2.794364996254444,
|
|
"epoch": 0.21145124716553287,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.248046875,
|
|
"kl": 0.001845422304540989,
|
|
"learning_rate": 7.891156462585033e-07,
|
|
"loss": -0.0219,
|
|
"num_tokens": 35257919.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 373,
|
|
"step_time": 293.63794895634055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3577.0,
|
|
"completions/mean_length": 1372.796875,
|
|
"completions/mean_terminated_length": 744.3654174804688,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.4559489972889423,
|
|
"epoch": 0.21201814058956917,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.21484375,
|
|
"kl": 0.001547759529785253,
|
|
"learning_rate": 7.885487528344671e-07,
|
|
"loss": 0.0102,
|
|
"num_tokens": 35357138.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 374,
|
|
"step_time": 296.37745548412204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3983.0,
|
|
"completions/mean_length": 1502.6875,
|
|
"completions/mean_terminated_length": 964.4528198242188,
|
|
"completions/min_length": 164.0,
|
|
"completions/min_terminated_length": 164.0,
|
|
"entropy": 3.559647735208273,
|
|
"epoch": 0.21258503401360543,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1298828125,
|
|
"kl": 0.0020263616788724903,
|
|
"learning_rate": 7.879818594104308e-07,
|
|
"loss": 0.0046,
|
|
"num_tokens": 35464750.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 375,
|
|
"step_time": 294.14539925567806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3554.0,
|
|
"completions/mean_length": 1158.1875,
|
|
"completions/mean_terminated_length": 738.5000610351562,
|
|
"completions/min_length": 140.0,
|
|
"completions/min_terminated_length": 140.0,
|
|
"entropy": 2.1302271597087383,
|
|
"epoch": 0.21315192743764172,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.0016697567079972941,
|
|
"learning_rate": 7.874149659863946e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 35550202.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5883605480194092,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 376,
|
|
"step_time": 294.99223101790994
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3483.0,
|
|
"completions/mean_length": 1143.125,
|
|
"completions/mean_terminated_length": 892.88134765625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.006809718906879,
|
|
"epoch": 0.213718820861678,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1943359375,
|
|
"kl": 0.0018984173857461428,
|
|
"learning_rate": 7.868480725623582e-07,
|
|
"loss": 0.0186,
|
|
"num_tokens": 35633106.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 377,
|
|
"step_time": 290.6259912196547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2856.0,
|
|
"completions/mean_length": 1040.265625,
|
|
"completions/mean_terminated_length": 724.1551513671875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.1822089068591595,
|
|
"epoch": 0.21428571428571427,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.0018458862286934163,
|
|
"learning_rate": 7.86281179138322e-07,
|
|
"loss": 0.0425,
|
|
"num_tokens": 35711123.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 378,
|
|
"step_time": 294.7133179754019
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4025.0,
|
|
"completions/mean_length": 1367.15625,
|
|
"completions/mean_terminated_length": 920.6181640625,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 4.002276659011841,
|
|
"epoch": 0.21485260770975056,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3046875,
|
|
"kl": 0.0013462407514452934,
|
|
"learning_rate": 7.857142857142856e-07,
|
|
"loss": 0.0192,
|
|
"num_tokens": 35811501.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 379,
|
|
"step_time": 299.2446444295347
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4073.0,
|
|
"completions/mean_length": 1600.015625,
|
|
"completions/mean_terminated_length": 963.7843627929688,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 3.1706656143069267,
|
|
"epoch": 0.21541950113378686,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.208984375,
|
|
"kl": 0.0026700084927142598,
|
|
"learning_rate": 7.851473922902494e-07,
|
|
"loss": -0.0208,
|
|
"num_tokens": 35929654.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 380,
|
|
"step_time": 308.2830661116168
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4096.0,
|
|
"completions/mean_length": 1219.4375,
|
|
"completions/mean_terminated_length": 921.862060546875,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"entropy": 3.471789874136448,
|
|
"epoch": 0.21598639455782312,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.0013272836004034616,
|
|
"learning_rate": 7.845804988662131e-07,
|
|
"loss": 0.0727,
|
|
"num_tokens": 36019194.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 381,
|
|
"step_time": 307.0177404200658
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4085.0,
|
|
"completions/mean_length": 809.609375,
|
|
"completions/mean_terminated_length": 647.9835815429688,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.630410388112068,
|
|
"epoch": 0.2165532879818594,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.0020975194602215197,
|
|
"learning_rate": 7.840136054421769e-07,
|
|
"loss": 0.1147,
|
|
"num_tokens": 36082337.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 382,
|
|
"step_time": 295.45017794612795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3947.0,
|
|
"completions/mean_length": 1296.328125,
|
|
"completions/mean_terminated_length": 952.5087890625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 4.066781774163246,
|
|
"epoch": 0.2171201814058957,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2236328125,
|
|
"kl": 0.0011764016635424923,
|
|
"learning_rate": 7.834467120181405e-07,
|
|
"loss": 0.0375,
|
|
"num_tokens": 36177758.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 383,
|
|
"step_time": 306.97376155201346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3918.0,
|
|
"completions/mean_length": 1130.859375,
|
|
"completions/mean_terminated_length": 766.7192993164062,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"entropy": 2.8013731352984905,
|
|
"epoch": 0.21768707482993196,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.001920601582241943,
|
|
"learning_rate": 7.828798185941043e-07,
|
|
"loss": -0.0019,
|
|
"num_tokens": 36263821.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 384,
|
|
"step_time": 302.87433452066034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3958.0,
|
|
"completions/mean_length": 1097.515625,
|
|
"completions/mean_terminated_length": 787.3275756835938,
|
|
"completions/min_length": 105.0,
|
|
"completions/min_terminated_length": 105.0,
|
|
"entropy": 3.3111561834812164,
|
|
"epoch": 0.21825396825396826,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.291015625,
|
|
"kl": 0.0017533823083795141,
|
|
"learning_rate": 7.82312925170068e-07,
|
|
"loss": 0.0183,
|
|
"num_tokens": 36345678.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7295067310333252,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 385,
|
|
"step_time": 301.5573451537639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3591.0,
|
|
"completions/mean_length": 1211.484375,
|
|
"completions/mean_terminated_length": 612.8113403320312,
|
|
"completions/min_length": 116.0,
|
|
"completions/min_terminated_length": 116.0,
|
|
"entropy": 2.12896141782403,
|
|
"epoch": 0.21882086167800455,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1845703125,
|
|
"kl": 0.002257652839034563,
|
|
"learning_rate": 7.817460317460318e-07,
|
|
"loss": 0.0186,
|
|
"num_tokens": 36439149.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 386,
|
|
"step_time": 322.2181938337162
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3369.0,
|
|
"completions/mean_length": 1139.0,
|
|
"completions/mean_terminated_length": 888.4067993164062,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.556562624871731,
|
|
"epoch": 0.2193877551020408,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2890625,
|
|
"kl": 0.002096428152071894,
|
|
"learning_rate": 7.811791383219954e-07,
|
|
"loss": 0.0817,
|
|
"num_tokens": 36525397.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5883605480194092,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 387,
|
|
"step_time": 301.1546203037724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4021.0,
|
|
"completions/mean_length": 1397.546875,
|
|
"completions/mean_terminated_length": 1066.157958984375,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 3.776941567659378,
|
|
"epoch": 0.2199546485260771,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.236328125,
|
|
"kl": 0.0012676407204708084,
|
|
"learning_rate": 7.806122448979591e-07,
|
|
"loss": 0.0388,
|
|
"num_tokens": 36626040.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6662480235099792,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 388,
|
|
"step_time": 295.3469817014411
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3844.0,
|
|
"completions/mean_length": 1362.78125,
|
|
"completions/mean_terminated_length": 795.5094604492188,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.865013089030981,
|
|
"epoch": 0.2205215419501134,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2265625,
|
|
"kl": 0.00144556881787139,
|
|
"learning_rate": 7.800453514739229e-07,
|
|
"loss": 0.0219,
|
|
"num_tokens": 36726586.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 389,
|
|
"step_time": 296.1360622383654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3881.0,
|
|
"completions/mean_length": 1110.765625,
|
|
"completions/mean_terminated_length": 911.7500610351562,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.1485432758927345,
|
|
"epoch": 0.22108843537414966,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.21875,
|
|
"kl": 0.0012700370098173153,
|
|
"learning_rate": 7.794784580498865e-07,
|
|
"loss": -0.0158,
|
|
"num_tokens": 36808931.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 390,
|
|
"step_time": 293.5197555180639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4037.0,
|
|
"completions/mean_length": 798.921875,
|
|
"completions/mean_terminated_length": 579.11669921875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.0194799602031708,
|
|
"epoch": 0.22165532879818595,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.404296875,
|
|
"kl": 0.0020651695485867094,
|
|
"learning_rate": 7.789115646258503e-07,
|
|
"loss": 0.0445,
|
|
"num_tokens": 36871334.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 391,
|
|
"step_time": 292.6065103439614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3644.0,
|
|
"completions/mean_length": 887.375,
|
|
"completions/mean_terminated_length": 729.57373046875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.4883970953524113,
|
|
"epoch": 0.2222222222222222,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.287109375,
|
|
"kl": 0.0020892868415103294,
|
|
"learning_rate": 7.78344671201814e-07,
|
|
"loss": 0.0372,
|
|
"num_tokens": 36939526.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 392,
|
|
"step_time": 292.00554332882166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4071.0,
|
|
"completions/mean_length": 1419.390625,
|
|
"completions/mean_terminated_length": 1037.0179443359375,
|
|
"completions/min_length": 116.0,
|
|
"completions/min_terminated_length": 116.0,
|
|
"entropy": 4.893246464431286,
|
|
"epoch": 0.2227891156462585,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.30859375,
|
|
"kl": 0.001233271841556416,
|
|
"learning_rate": 7.777777777777778e-07,
|
|
"loss": -0.0503,
|
|
"num_tokens": 37041447.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.6267831921577454,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 393,
|
|
"step_time": 295.95994062069803
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3812.0,
|
|
"completions/mean_length": 1084.953125,
|
|
"completions/mean_terminated_length": 936.8687744140625,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 3.491551138460636,
|
|
"epoch": 0.2233560090702948,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.39453125,
|
|
"kl": 0.0019169310544384643,
|
|
"learning_rate": 7.772108843537415e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 37123420.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 394,
|
|
"step_time": 300.3087148098275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3717.0,
|
|
"completions/mean_length": 1581.21875,
|
|
"completions/mean_terminated_length": 940.1961059570312,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 4.595789223909378,
|
|
"epoch": 0.22392290249433106,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.140625,
|
|
"kl": 0.0017633351071708603,
|
|
"learning_rate": 7.766439909297052e-07,
|
|
"loss": 0.012,
|
|
"num_tokens": 37236626.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 395,
|
|
"step_time": 300.34654100984335
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3981.0,
|
|
"completions/mean_length": 1304.671875,
|
|
"completions/mean_terminated_length": 1015.913818359375,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 4.356452081352472,
|
|
"epoch": 0.22448979591836735,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.271484375,
|
|
"kl": 0.0014264194014685927,
|
|
"learning_rate": 7.760770975056688e-07,
|
|
"loss": 0.0583,
|
|
"num_tokens": 37332269.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6782841682434082,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 396,
|
|
"step_time": 303.81499348487705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3467.0,
|
|
"completions/mean_length": 720.453125,
|
|
"completions/mean_terminated_length": 666.873046875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.8045205511152744,
|
|
"epoch": 0.22505668934240364,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.23828125,
|
|
"kl": 0.0016409798354288796,
|
|
"learning_rate": 7.755102040816326e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 37388594.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4753340482711792,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 397,
|
|
"step_time": 293.10425563156605
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4065.0,
|
|
"completions/mean_length": 1258.9375,
|
|
"completions/mean_terminated_length": 853.6428833007812,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.4828469827771187,
|
|
"epoch": 0.2256235827664399,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.306640625,
|
|
"kl": 0.0014735936601937283,
|
|
"learning_rate": 7.749433106575963e-07,
|
|
"loss": 0.0701,
|
|
"num_tokens": 37482614.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 398,
|
|
"step_time": 299.26312240678817
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3801.0,
|
|
"completions/mean_length": 1303.6875,
|
|
"completions/mean_terminated_length": 1014.8275756835938,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 4.619399968534708,
|
|
"epoch": 0.2261904761904762,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2890625,
|
|
"kl": 0.001796510121494066,
|
|
"learning_rate": 7.743764172335601e-07,
|
|
"loss": -0.0496,
|
|
"num_tokens": 37577410.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6412084698677063,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 399,
|
|
"step_time": 295.9423144198954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3935.0,
|
|
"completions/mean_length": 941.234375,
|
|
"completions/mean_terminated_length": 786.0819091796875,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 2.890203081071377,
|
|
"epoch": 0.22675736961451248,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.0014154092605167534,
|
|
"learning_rate": 7.738095238095238e-07,
|
|
"loss": 0.0658,
|
|
"num_tokens": 37647273.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 400,
|
|
"step_time": 288.85523932240903
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2998.0,
|
|
"completions/mean_length": 1129.484375,
|
|
"completions/mean_terminated_length": 822.6034545898438,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 3.657008931040764,
|
|
"epoch": 0.22732426303854875,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.205078125,
|
|
"kl": 0.0014798003430769313,
|
|
"learning_rate": 7.732426303854876e-07,
|
|
"loss": 0.012,
|
|
"num_tokens": 37732472.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 401,
|
|
"step_time": 298.95484398212284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3958.0,
|
|
"completions/mean_length": 1498.96875,
|
|
"completions/mean_terminated_length": 1018.0370483398438,
|
|
"completions/min_length": 44.0,
|
|
"completions/min_terminated_length": 44.0,
|
|
"entropy": 3.3186546824872494,
|
|
"epoch": 0.22789115646258504,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.0013445516051433515,
|
|
"learning_rate": 7.726757369614512e-07,
|
|
"loss": 0.0539,
|
|
"num_tokens": 37842662.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 402,
|
|
"step_time": 302.0707771368325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3225.0,
|
|
"completions/mean_length": 1102.53125,
|
|
"completions/mean_terminated_length": 612.69091796875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.0880488380789757,
|
|
"epoch": 0.22845804988662133,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.0020504709355009254,
|
|
"learning_rate": 7.721088435374149e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 37923744.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5890451669692993,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 403,
|
|
"step_time": 292.8356831539422
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3901.0,
|
|
"completions/mean_length": 1096.15625,
|
|
"completions/mean_terminated_length": 896.1666870117188,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.1105887927114964,
|
|
"epoch": 0.2290249433106576,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.001272784222237533,
|
|
"learning_rate": 7.715419501133786e-07,
|
|
"loss": 0.0298,
|
|
"num_tokens": 38006754.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 404,
|
|
"step_time": 304.97734472341835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3833.0,
|
|
"completions/mean_length": 1515.21875,
|
|
"completions/mean_terminated_length": 857.37255859375,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 4.143358360975981,
|
|
"epoch": 0.22959183673469388,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1767578125,
|
|
"kl": 0.002094690022204304,
|
|
"learning_rate": 7.709750566893424e-07,
|
|
"loss": 0.0119,
|
|
"num_tokens": 38117344.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 405,
|
|
"step_time": 297.9922364205122
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3902.0,
|
|
"completions/mean_length": 974.28125,
|
|
"completions/mean_terminated_length": 873.5806274414062,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.750041037797928,
|
|
"epoch": 0.23015873015873015,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.353515625,
|
|
"kl": 0.0013144648346496979,
|
|
"learning_rate": 7.704081632653061e-07,
|
|
"loss": 0.01,
|
|
"num_tokens": 38189906.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.6784212589263916,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 406,
|
|
"step_time": 294.31997813936323
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4061.0,
|
|
"completions/mean_length": 1544.671875,
|
|
"completions/mean_terminated_length": 1015.1509399414062,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 2.97406118363142,
|
|
"epoch": 0.23072562358276644,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.001828713609938859,
|
|
"learning_rate": 7.698412698412699e-07,
|
|
"loss": 0.0481,
|
|
"num_tokens": 38301269.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 407,
|
|
"step_time": 298.18998036999255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4067.0,
|
|
"completions/mean_length": 1311.453125,
|
|
"completions/mean_terminated_length": 913.6607666015625,
|
|
"completions/min_length": 30.0,
|
|
"completions/min_terminated_length": 30.0,
|
|
"entropy": 3.1563703641295433,
|
|
"epoch": 0.23129251700680273,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.248046875,
|
|
"kl": 0.001512056820502039,
|
|
"learning_rate": 7.692743764172335e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 38395658.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 408,
|
|
"step_time": 292.98286969866604
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3982.0,
|
|
"completions/mean_length": 1415.984375,
|
|
"completions/mean_terminated_length": 1033.125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.2954092267900705,
|
|
"epoch": 0.231859410430839,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.0015296718975150725,
|
|
"learning_rate": 7.687074829931972e-07,
|
|
"loss": -0.076,
|
|
"num_tokens": 38498889.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6664806604385376,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 409,
|
|
"step_time": 296.84177739638835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3918.0,
|
|
"completions/mean_length": 1194.25,
|
|
"completions/mean_terminated_length": 1000.800048828125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.125131338834763,
|
|
"epoch": 0.23242630385487528,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.0014890253532939823,
|
|
"learning_rate": 7.68140589569161e-07,
|
|
"loss": 0.0111,
|
|
"num_tokens": 38589425.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 410,
|
|
"step_time": 318.92869498115033
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3369.0,
|
|
"completions/mean_length": 933.71875,
|
|
"completions/mean_terminated_length": 665.7288208007812,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 2.0793898813426495,
|
|
"epoch": 0.23299319727891157,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.322265625,
|
|
"kl": 0.001860350399510935,
|
|
"learning_rate": 7.675736961451246e-07,
|
|
"loss": 0.0908,
|
|
"num_tokens": 38661255.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 411,
|
|
"step_time": 300.06950910016894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3551.0,
|
|
"completions/mean_length": 1163.859375,
|
|
"completions/mean_terminated_length": 803.7719116210938,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.3974614329636097,
|
|
"epoch": 0.23356009070294784,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.002429354473861167,
|
|
"learning_rate": 7.670068027210884e-07,
|
|
"loss": -0.0233,
|
|
"num_tokens": 38746166.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.7549029588699341,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 412,
|
|
"step_time": 293.9550670655444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4083.0,
|
|
"completions/mean_length": 1285.171875,
|
|
"completions/mean_terminated_length": 825.2181396484375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.517598506063223,
|
|
"epoch": 0.23412698412698413,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.0017115061400545528,
|
|
"learning_rate": 7.664399092970521e-07,
|
|
"loss": 0.0288,
|
|
"num_tokens": 38838617.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 413,
|
|
"step_time": 290.1487058866769
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3951.0,
|
|
"completions/mean_length": 1313.15625,
|
|
"completions/mean_terminated_length": 857.7817993164062,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 2.6312940698117018,
|
|
"epoch": 0.23469387755102042,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.23828125,
|
|
"kl": 0.0016931954687606776,
|
|
"learning_rate": 7.658730158730159e-07,
|
|
"loss": -0.0279,
|
|
"num_tokens": 38934019.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 414,
|
|
"step_time": 296.73214874975383
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3842.0,
|
|
"completions/mean_length": 1059.296875,
|
|
"completions/mean_terminated_length": 686.368408203125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.0324822701513767,
|
|
"epoch": 0.23526077097505668,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2109375,
|
|
"kl": 0.0022498586713481927,
|
|
"learning_rate": 7.653061224489795e-07,
|
|
"loss": 0.0614,
|
|
"num_tokens": 39013902.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 415,
|
|
"step_time": 297.1698560230434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3978.0,
|
|
"completions/mean_length": 1172.1875,
|
|
"completions/mean_terminated_length": 924.4067993164062,
|
|
"completions/min_length": 34.0,
|
|
"completions/min_terminated_length": 34.0,
|
|
"entropy": 3.493779791519046,
|
|
"epoch": 0.23582766439909297,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.17578125,
|
|
"kl": 0.0016595613633398898,
|
|
"learning_rate": 7.647392290249433e-07,
|
|
"loss": 0.0295,
|
|
"num_tokens": 39102850.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.8648432493209839,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 416,
|
|
"step_time": 302.1476899283007
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4090.0,
|
|
"completions/mean_length": 1529.375,
|
|
"completions/mean_terminated_length": 996.6792602539062,
|
|
"completions/min_length": 159.0,
|
|
"completions/min_terminated_length": 159.0,
|
|
"entropy": 4.744919311255217,
|
|
"epoch": 0.23639455782312926,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.134765625,
|
|
"kl": 0.0017254153863177635,
|
|
"learning_rate": 7.64172335600907e-07,
|
|
"loss": 0.0176,
|
|
"num_tokens": 39213930.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4714045226573944,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 417,
|
|
"step_time": 297.82898204773664
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3593.0,
|
|
"completions/mean_length": 982.546875,
|
|
"completions/mean_terminated_length": 718.6949462890625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 2.9274300076067448,
|
|
"epoch": 0.23696145124716553,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.0019668883469421417,
|
|
"learning_rate": 7.636054421768708e-07,
|
|
"loss": 0.0132,
|
|
"num_tokens": 39289413.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 418,
|
|
"step_time": 301.8002496920526
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3302.0,
|
|
"completions/mean_length": 1272.078125,
|
|
"completions/mean_terminated_length": 685.9811401367188,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"entropy": 2.2984911371022463,
|
|
"epoch": 0.23752834467120182,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2412109375,
|
|
"kl": 0.001998718649701914,
|
|
"learning_rate": 7.630385487528344e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 39382370.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.41277948021888733,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 419,
|
|
"step_time": 294.1703432239592
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3649.0,
|
|
"completions/mean_length": 1323.515625,
|
|
"completions/mean_terminated_length": 983.0350952148438,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 4.132300119847059,
|
|
"epoch": 0.23809523809523808,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.322265625,
|
|
"kl": 0.0013739411515416577,
|
|
"learning_rate": 7.624716553287982e-07,
|
|
"loss": 0.0231,
|
|
"num_tokens": 39479235.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 420,
|
|
"step_time": 295.6117933075875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3990.0,
|
|
"completions/mean_length": 1386.859375,
|
|
"completions/mean_terminated_length": 885.1666870117188,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.570806212723255,
|
|
"epoch": 0.23866213151927437,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2412109375,
|
|
"kl": 0.0014071986370254308,
|
|
"learning_rate": 7.619047619047618e-07,
|
|
"loss": 0.0175,
|
|
"num_tokens": 39579754.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 421,
|
|
"step_time": 294.0098496126011
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3329.0,
|
|
"completions/mean_length": 1107.296875,
|
|
"completions/mean_terminated_length": 798.1206665039062,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.599139340221882,
|
|
"epoch": 0.23922902494331066,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.34375,
|
|
"kl": 0.001797916227587848,
|
|
"learning_rate": 7.613378684807256e-07,
|
|
"loss": -0.0228,
|
|
"num_tokens": 39661421.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5274578928947449,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 422,
|
|
"step_time": 293.1747271474451
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3955.0,
|
|
"completions/mean_length": 1037.703125,
|
|
"completions/mean_terminated_length": 778.5254516601562,
|
|
"completions/min_length": 122.0,
|
|
"completions/min_terminated_length": 122.0,
|
|
"entropy": 3.3564374186098576,
|
|
"epoch": 0.23979591836734693,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.21875,
|
|
"kl": 0.0013273896947794128,
|
|
"learning_rate": 7.607709750566893e-07,
|
|
"loss": -0.0055,
|
|
"num_tokens": 39739746.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 423,
|
|
"step_time": 295.4561696825549
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3359.0,
|
|
"completions/mean_length": 1113.25,
|
|
"completions/mean_terminated_length": 860.4745483398438,
|
|
"completions/min_length": 26.0,
|
|
"completions/min_terminated_length": 26.0,
|
|
"entropy": 3.37971917912364,
|
|
"epoch": 0.24036281179138322,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.3984375,
|
|
"kl": 0.001881603437141166,
|
|
"learning_rate": 7.602040816326531e-07,
|
|
"loss": 0.0556,
|
|
"num_tokens": 39822450.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 424,
|
|
"step_time": 293.7390189040452
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3140.0,
|
|
"completions/mean_length": 1238.84375,
|
|
"completions/mean_terminated_length": 771.30908203125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 2.6694610193371773,
|
|
"epoch": 0.2409297052154195,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2294921875,
|
|
"kl": 0.002417201538264635,
|
|
"learning_rate": 7.596371882086168e-07,
|
|
"loss": 0.0354,
|
|
"num_tokens": 39914328.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 425,
|
|
"step_time": 297.5649088798091
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3497.0,
|
|
"completions/mean_length": 1007.765625,
|
|
"completions/mean_terminated_length": 688.2930908203125,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"entropy": 3.1933873035013676,
|
|
"epoch": 0.24149659863945577,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2138671875,
|
|
"kl": 0.0018550817931100028,
|
|
"learning_rate": 7.590702947845806e-07,
|
|
"loss": -0.0315,
|
|
"num_tokens": 39993041.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.666620135307312,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 426,
|
|
"step_time": 319.5642735818401
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4004.0,
|
|
"completions/mean_length": 1179.046875,
|
|
"completions/mean_terminated_length": 820.8245849609375,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.247973505407572,
|
|
"epoch": 0.24206349206349206,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.0020758294776896946,
|
|
"learning_rate": 7.585034013605441e-07,
|
|
"loss": 0.0782,
|
|
"num_tokens": 40081708.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.7014724016189575,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 427,
|
|
"step_time": 304.9623340750113
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3750.0,
|
|
"completions/mean_length": 1334.765625,
|
|
"completions/mean_terminated_length": 940.3035888671875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.298788692802191,
|
|
"epoch": 0.24263038548752835,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1748046875,
|
|
"kl": 0.0012791893022949807,
|
|
"learning_rate": 7.579365079365078e-07,
|
|
"loss": 0.0129,
|
|
"num_tokens": 40179269.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 428,
|
|
"step_time": 299.1383136184886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3310.0,
|
|
"completions/mean_length": 1314.578125,
|
|
"completions/mean_terminated_length": 799.5,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.217808473855257,
|
|
"epoch": 0.24319727891156462,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2265625,
|
|
"kl": 0.0015218931566778338,
|
|
"learning_rate": 7.573696145124716e-07,
|
|
"loss": 0.026,
|
|
"num_tokens": 40274234.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 429,
|
|
"step_time": 292.306060356088
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4059.0,
|
|
"completions/mean_length": 1422.6875,
|
|
"completions/mean_terminated_length": 927.629638671875,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 2.6152460537850857,
|
|
"epoch": 0.2437641723356009,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.16015625,
|
|
"kl": 0.0012690617950283922,
|
|
"learning_rate": 7.568027210884353e-07,
|
|
"loss": -0.0062,
|
|
"num_tokens": 40379126.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 430,
|
|
"step_time": 299.54230482690036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3615.0,
|
|
"completions/mean_length": 932.734375,
|
|
"completions/mean_terminated_length": 721.8500366210938,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.814725812524557,
|
|
"epoch": 0.2443310657596372,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.419921875,
|
|
"kl": 0.0026313944181310944,
|
|
"learning_rate": 7.562358276643991e-07,
|
|
"loss": 0.0156,
|
|
"num_tokens": 40449701.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 431,
|
|
"step_time": 294.5451189512387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3927.0,
|
|
"completions/mean_length": 1063.9375,
|
|
"completions/mean_terminated_length": 914.8196411132812,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 2.978424996137619,
|
|
"epoch": 0.24489795918367346,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.419921875,
|
|
"kl": 0.0019791342401731526,
|
|
"learning_rate": 7.556689342403628e-07,
|
|
"loss": 0.1045,
|
|
"num_tokens": 40528257.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 432,
|
|
"step_time": 293.1097595356405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2882.0,
|
|
"completions/mean_length": 771.5625,
|
|
"completions/mean_terminated_length": 608.0655517578125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 1.9814389497041702,
|
|
"epoch": 0.24546485260770975,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.478515625,
|
|
"kl": 0.0018759423110168427,
|
|
"learning_rate": 7.551020408163265e-07,
|
|
"loss": -0.033,
|
|
"num_tokens": 40590053.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.7759642601013184,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 433,
|
|
"step_time": 310.92460263706744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3897.0,
|
|
"completions/mean_length": 1108.859375,
|
|
"completions/mean_terminated_length": 799.8448486328125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.3193601071834564,
|
|
"epoch": 0.24603174603174602,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.453125,
|
|
"kl": 0.0019465246878098696,
|
|
"learning_rate": 7.545351473922901e-07,
|
|
"loss": -0.1088,
|
|
"num_tokens": 40680636.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.666620135307312,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 434,
|
|
"step_time": 346.41499474365264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3182.0,
|
|
"completions/mean_length": 1367.8125,
|
|
"completions/mean_terminated_length": 978.0714721679688,
|
|
"completions/min_length": 19.0,
|
|
"completions/min_terminated_length": 19.0,
|
|
"entropy": 3.699978746473789,
|
|
"epoch": 0.2465986394557823,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.0017686764949758071,
|
|
"learning_rate": 7.539682539682539e-07,
|
|
"loss": 0.0306,
|
|
"num_tokens": 40778840.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 435,
|
|
"step_time": 298.42550599202514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3836.0,
|
|
"completions/mean_length": 1504.03125,
|
|
"completions/mean_terminated_length": 1185.7193603515625,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 3.9608581140637398,
|
|
"epoch": 0.2471655328798186,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.1806640625,
|
|
"kl": 0.001418873338479898,
|
|
"learning_rate": 7.534013605442176e-07,
|
|
"loss": 0.0507,
|
|
"num_tokens": 40885506.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 436,
|
|
"step_time": 294.686938627623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3131.0,
|
|
"completions/mean_length": 1098.296875,
|
|
"completions/mean_terminated_length": 730.1578979492188,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 2.6076142862439156,
|
|
"epoch": 0.24773242630385486,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.0018644657066033687,
|
|
"learning_rate": 7.528344671201814e-07,
|
|
"loss": -0.0304,
|
|
"num_tokens": 40966829.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 437,
|
|
"step_time": 301.3715253164992
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3701.0,
|
|
"completions/mean_length": 1595.5625,
|
|
"completions/mean_terminated_length": 1076.603759765625,
|
|
"completions/min_length": 101.0,
|
|
"completions/min_terminated_length": 101.0,
|
|
"entropy": 2.996962059289217,
|
|
"epoch": 0.24829931972789115,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.2255859375,
|
|
"kl": 0.0010735964206105564,
|
|
"learning_rate": 7.522675736961451e-07,
|
|
"loss": 0.0582,
|
|
"num_tokens": 41080169.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5261042714118958,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 438,
|
|
"step_time": 292.6421411121264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4054.0,
|
|
"completions/mean_length": 812.8125,
|
|
"completions/mean_terminated_length": 760.698486328125,
|
|
"completions/min_length": 29.0,
|
|
"completions/min_terminated_length": 29.0,
|
|
"entropy": 2.873863724991679,
|
|
"epoch": 0.24886621315192745,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.333984375,
|
|
"kl": 0.0016787676504463889,
|
|
"learning_rate": 7.517006802721088e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 41144797.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 439,
|
|
"step_time": 295.166032015346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3724.0,
|
|
"completions/mean_length": 1005.078125,
|
|
"completions/mean_terminated_length": 743.1356201171875,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"entropy": 3.237634778022766,
|
|
"epoch": 0.2494331065759637,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.271484375,
|
|
"kl": 0.0020494155378401047,
|
|
"learning_rate": 7.511337868480725e-07,
|
|
"loss": 0.0711,
|
|
"num_tokens": 41220354.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 440,
|
|
"step_time": 298.98595599643886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3787.0,
|
|
"completions/mean_length": 1482.34375,
|
|
"completions/mean_terminated_length": 1108.96435546875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.5558236204087734,
|
|
"epoch": 0.25,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1796875,
|
|
"kl": 0.001472385167289758,
|
|
"learning_rate": 7.505668934240363e-07,
|
|
"loss": 0.0118,
|
|
"num_tokens": 41327408.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 441,
|
|
"step_time": 299.76319946255535
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 1764,
|
|
"num_input_tokens_seen": 41327408,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|