0a395da4f3
Model: azalahmadkhan/Llama-3.2-3B-Instruct-GRPO-vanilla-G-4-novllm-50pct Source: Original Platform
6635 lines
245 KiB
JSON
6635 lines
245 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.4988662131519274,
|
|
"eval_steps": 500,
|
|
"global_step": 220,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3681.0,
|
|
"completions/mean_length": 1268.2265625,
|
|
"completions/mean_terminated_length": 980.7562866210938,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"entropy": 4.420659868046641,
|
|
"epoch": 0.0022675736961451248,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.19921875,
|
|
"kl": 0.0003264336364736664,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0093,
|
|
"num_tokens": 184821.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31472553312778473,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 1,
|
|
"step_time": 600.7805640818551
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3984.0,
|
|
"completions/mean_length": 1622.3203125,
|
|
"completions/mean_terminated_length": 962.6857299804688,
|
|
"completions/min_length": 107.5,
|
|
"completions/min_terminated_length": 107.5,
|
|
"entropy": 3.2865531742572784,
|
|
"epoch": 0.0045351473922902496,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.09375,
|
|
"kl": 0.0003618000596361526,
|
|
"learning_rate": 9.977324263038547e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 417594.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2519752085208893,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 2,
|
|
"step_time": 603.3548571760766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3468.5,
|
|
"completions/mean_length": 1188.703125,
|
|
"completions/mean_terminated_length": 803.1010437011719,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 2.9000580459833145,
|
|
"epoch": 0.006802721088435374,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.228515625,
|
|
"kl": 0.0003971439623455808,
|
|
"learning_rate": 9.954648526077097e-07,
|
|
"loss": -0.0146,
|
|
"num_tokens": 592512.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.2943021357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 3,
|
|
"step_time": 592.7558827972971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3806.5,
|
|
"completions/mean_length": 1708.2734375,
|
|
"completions/mean_terminated_length": 1070.256103515625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.978165291249752,
|
|
"epoch": 0.009070294784580499,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.134765625,
|
|
"kl": 0.0003516419765219325,
|
|
"learning_rate": 9.931972789115645e-07,
|
|
"loss": 0.0028,
|
|
"num_tokens": 835279.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 4,
|
|
"step_time": 592.1812793556601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3869.0,
|
|
"completions/mean_length": 1158.1484375,
|
|
"completions/mean_terminated_length": 680.8644409179688,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.368096474558115,
|
|
"epoch": 0.011337868480725623,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.00045219186085887486,
|
|
"learning_rate": 9.909297052154193e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 1006302.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.2582981809973717,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 5,
|
|
"step_time": 594.9633022211492
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3758.0,
|
|
"completions/mean_length": 1112.7421875,
|
|
"completions/mean_terminated_length": 805.9321899414062,
|
|
"completions/min_length": 1.5,
|
|
"completions/min_terminated_length": 1.5,
|
|
"entropy": 3.4849978666752577,
|
|
"epoch": 0.013605442176870748,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.0004078899446540163,
|
|
"learning_rate": 9.886621315192743e-07,
|
|
"loss": 0.026,
|
|
"num_tokens": 1177829.0,
|
|
"reward": 0.14453125,
|
|
"reward_std": 0.5158129632472992,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48353175818920135,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 6,
|
|
"step_time": 635.190734772943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3845.5,
|
|
"completions/mean_length": 1290.9140625,
|
|
"completions/mean_terminated_length": 861.3218994140625,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.505165945738554,
|
|
"epoch": 0.015873015873015872,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.00037227333541522967,
|
|
"learning_rate": 9.86394557823129e-07,
|
|
"loss": 0.035,
|
|
"num_tokens": 1365610.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3109140843153,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 7,
|
|
"step_time": 591.5275562573224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4059.5,
|
|
"completions/mean_length": 1521.6640625,
|
|
"completions/mean_terminated_length": 1102.5285034179688,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.9575814120471478,
|
|
"epoch": 0.018140589569160998,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1513671875,
|
|
"kl": 0.0003584225587474066,
|
|
"learning_rate": 9.84126984126984e-07,
|
|
"loss": 0.0205,
|
|
"num_tokens": 1586235.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.20868311822414398,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 8,
|
|
"step_time": 610.3726020189933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3902.5,
|
|
"completions/mean_length": 1471.5859375,
|
|
"completions/mean_terminated_length": 957.7996215820312,
|
|
"completions/min_length": 25.5,
|
|
"completions/min_terminated_length": 25.5,
|
|
"entropy": 3.6384904142469168,
|
|
"epoch": 0.02040816326530612,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2236328125,
|
|
"kl": 0.00037756811252620537,
|
|
"learning_rate": 9.818594104308389e-07,
|
|
"loss": 0.0225,
|
|
"num_tokens": 1797770.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3733227699995041,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 9,
|
|
"step_time": 596.3971076705493
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3439.0,
|
|
"completions/mean_length": 1379.828125,
|
|
"completions/mean_terminated_length": 848.1103515625,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 4.184969509020448,
|
|
"epoch": 0.022675736961451247,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.00037742951417385484,
|
|
"learning_rate": 9.795918367346939e-07,
|
|
"loss": 0.0448,
|
|
"num_tokens": 2000228.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.40065382421016693,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 10,
|
|
"step_time": 614.9045870057307
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4059.5,
|
|
"completions/mean_length": 1210.9296875,
|
|
"completions/mean_terminated_length": 853.6432800292969,
|
|
"completions/min_length": 54.5,
|
|
"completions/min_terminated_length": 54.5,
|
|
"entropy": 2.7617363315075636,
|
|
"epoch": 0.024943310657596373,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.00043528827609407017,
|
|
"learning_rate": 9.773242630385487e-07,
|
|
"loss": -0.0082,
|
|
"num_tokens": 2178859.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3079812675714493,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.06099375709891319,
|
|
"step": 11,
|
|
"step_time": 599.1445474587381
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3483.5,
|
|
"completions/mean_length": 1608.015625,
|
|
"completions/mean_terminated_length": 998.5797119140625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.71538058295846,
|
|
"epoch": 0.027210884353741496,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.130859375,
|
|
"kl": 0.0003708188855853223,
|
|
"learning_rate": 9.750566893424036e-07,
|
|
"loss": 0.0254,
|
|
"num_tokens": 2409449.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3062363266944885,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 12,
|
|
"step_time": 604.6724895187654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3624.5,
|
|
"completions/mean_length": 1378.3046875,
|
|
"completions/mean_terminated_length": 844.677490234375,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 3.3661539182066917,
|
|
"epoch": 0.02947845804988662,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.220703125,
|
|
"kl": 0.00040208527752838563,
|
|
"learning_rate": 9.727891156462584e-07,
|
|
"loss": -0.0273,
|
|
"num_tokens": 2609844.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2628627270460129,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 13,
|
|
"step_time": 633.5494570918381
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4007.0,
|
|
"completions/mean_length": 1187.21875,
|
|
"completions/mean_terminated_length": 831.5615844726562,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"entropy": 3.0186092387884855,
|
|
"epoch": 0.031746031746031744,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1943359375,
|
|
"kl": 0.00042435936575202504,
|
|
"learning_rate": 9.705215419501134e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 2786412.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.42632731795310974,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 14,
|
|
"step_time": 591.7922107283957
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3788.5,
|
|
"completions/mean_length": 1438.6640625,
|
|
"completions/mean_terminated_length": 831.2064819335938,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.133930031210184,
|
|
"epoch": 0.034013605442176874,
|
|
"frac_reward_zero_std": 0.96875,
|
|
"grad_norm": 0.080078125,
|
|
"kl": 0.0004019950811198214,
|
|
"learning_rate": 9.682539682539682e-07,
|
|
"loss": -0.0017,
|
|
"num_tokens": 2995009.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.125,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 15,
|
|
"step_time": 606.26203110395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4070.0,
|
|
"completions/mean_length": 1565.796875,
|
|
"completions/mean_terminated_length": 1040.6603698730469,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 4.076895218342543,
|
|
"epoch": 0.036281179138321996,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.15234375,
|
|
"kl": 0.000358647768962328,
|
|
"learning_rate": 9.65986394557823e-07,
|
|
"loss": 0.0219,
|
|
"num_tokens": 3219087.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.17743311822414398,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 16,
|
|
"step_time": 600.7762890821323
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3746.5,
|
|
"completions/mean_length": 1242.6015625,
|
|
"completions/mean_terminated_length": 806.0835876464844,
|
|
"completions/min_length": 42.5,
|
|
"completions/min_terminated_length": 42.5,
|
|
"entropy": 3.07886391505599,
|
|
"epoch": 0.03854875283446712,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.0004230019103488303,
|
|
"learning_rate": 9.63718820861678e-07,
|
|
"loss": 0.0201,
|
|
"num_tokens": 3402980.0,
|
|
"reward": 0.02734375,
|
|
"reward_std": 0.17220696061849594,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 17,
|
|
"step_time": 593.2202473981306
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3938.0,
|
|
"completions/mean_length": 1397.1953125,
|
|
"completions/mean_terminated_length": 978.6914978027344,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 4.573173347860575,
|
|
"epoch": 0.04081632653061224,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.169921875,
|
|
"kl": 0.0003518174189593992,
|
|
"learning_rate": 9.614512471655328e-07,
|
|
"loss": -0.0097,
|
|
"num_tokens": 3604213.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3089591860771179,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 18,
|
|
"step_time": 588.3413797696121
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3858.5,
|
|
"completions/mean_length": 1294.984375,
|
|
"completions/mean_terminated_length": 868.6067504882812,
|
|
"completions/min_length": 59.0,
|
|
"completions/min_terminated_length": 59.0,
|
|
"entropy": 3.730927578173578,
|
|
"epoch": 0.04308390022675737,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.00038500537357322173,
|
|
"learning_rate": 9.591836734693876e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 3795731.0,
|
|
"reward": 0.18359375,
|
|
"reward_std": 0.5490992665290833,
|
|
"rewards/accuracy_reward_func/mean": 0.171875,
|
|
"rewards/accuracy_reward_func/std": 0.5463753789663315,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 19,
|
|
"step_time": 665.4319940893911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3696.5,
|
|
"completions/mean_length": 1467.2109375,
|
|
"completions/mean_terminated_length": 994.596923828125,
|
|
"completions/min_length": 39.0,
|
|
"completions/min_terminated_length": 39.0,
|
|
"entropy": 3.785406796261668,
|
|
"epoch": 0.045351473922902494,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.00038856124820085824,
|
|
"learning_rate": 9.569160997732426e-07,
|
|
"loss": 0.0419,
|
|
"num_tokens": 4007178.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.3037330359220505,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 20,
|
|
"step_time": 592.561503094621
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3836.5,
|
|
"completions/mean_length": 1551.0625,
|
|
"completions/mean_terminated_length": 1051.445556640625,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.296387929469347,
|
|
"epoch": 0.047619047619047616,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.0003285188749941881,
|
|
"learning_rate": 9.546485260770974e-07,
|
|
"loss": 0.0276,
|
|
"num_tokens": 4230118.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.3057980537414551,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 21,
|
|
"step_time": 619.5003909710795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3641.5,
|
|
"completions/mean_length": 1341.0703125,
|
|
"completions/mean_terminated_length": 892.7926330566406,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 3.226698711514473,
|
|
"epoch": 0.049886621315192746,
|
|
"frac_reward_zero_std": 0.96875,
|
|
"grad_norm": 0.1171875,
|
|
"kl": 0.0004377866116556106,
|
|
"learning_rate": 9.523809523809522e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 4431731.0,
|
|
"reward": 0.01953125,
|
|
"reward_std": 0.0922437533736229,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 22,
|
|
"step_time": 664.7066006427631
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3521.0,
|
|
"completions/mean_length": 1580.8203125,
|
|
"completions/mean_terminated_length": 1011.142578125,
|
|
"completions/min_length": 36.0,
|
|
"completions/min_terminated_length": 36.0,
|
|
"entropy": 3.4629636611789465,
|
|
"epoch": 0.05215419501133787,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.126953125,
|
|
"kl": 0.00039756272417434957,
|
|
"learning_rate": 9.501133786848072e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 4659060.0,
|
|
"reward": 0.02734375,
|
|
"reward_std": 0.17220696061849594,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 23,
|
|
"step_time": 609.2390451589599
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3862.5,
|
|
"completions/mean_length": 1417.40625,
|
|
"completions/mean_terminated_length": 857.968505859375,
|
|
"completions/min_length": 8.5,
|
|
"completions/min_terminated_length": 8.5,
|
|
"entropy": 3.3345618154853582,
|
|
"epoch": 0.05442176870748299,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.00041635158777353354,
|
|
"learning_rate": 9.47845804988662e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 4865088.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.4368836283683777,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.41934312880039215,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 24,
|
|
"step_time": 605.0491937622428
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3432.0,
|
|
"completions/mean_length": 1409.046875,
|
|
"completions/mean_terminated_length": 841.3591918945312,
|
|
"completions/min_length": 8.5,
|
|
"completions/min_terminated_length": 8.5,
|
|
"entropy": 3.0420125983655453,
|
|
"epoch": 0.05668934240362812,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.00044511446731121396,
|
|
"learning_rate": 9.45578231292517e-07,
|
|
"loss": -0.0466,
|
|
"num_tokens": 5070970.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.37423862516880035,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 25,
|
|
"step_time": 617.6828591921367
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4066.5,
|
|
"completions/mean_length": 1433.484375,
|
|
"completions/mean_terminated_length": 909.6171875,
|
|
"completions/min_length": 41.5,
|
|
"completions/min_terminated_length": 41.5,
|
|
"entropy": 3.7978090457618237,
|
|
"epoch": 0.05895691609977324,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.12890625,
|
|
"kl": 0.0003817280889961694,
|
|
"learning_rate": 9.433106575963718e-07,
|
|
"loss": 0.0094,
|
|
"num_tokens": 5277956.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.22862860560417175,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 26,
|
|
"step_time": 593.7618179973215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3716.0,
|
|
"completions/mean_length": 1305.46875,
|
|
"completions/mean_terminated_length": 918.2090148925781,
|
|
"completions/min_length": 94.0,
|
|
"completions/min_terminated_length": 94.0,
|
|
"entropy": 3.155247835442424,
|
|
"epoch": 0.061224489795918366,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.201171875,
|
|
"kl": 0.0004310551134949492,
|
|
"learning_rate": 9.410430839002267e-07,
|
|
"loss": 0.0454,
|
|
"num_tokens": 5467888.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.27729756385087967,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 27,
|
|
"step_time": 595.4528097324073
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4029.5,
|
|
"completions/mean_length": 1341.5078125,
|
|
"completions/mean_terminated_length": 801.0039978027344,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 3.1700076377019286,
|
|
"epoch": 0.06349206349206349,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1552734375,
|
|
"kl": 0.00045847328192394343,
|
|
"learning_rate": 9.387755102040816e-07,
|
|
"loss": 0.0124,
|
|
"num_tokens": 5662897.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3062363266944885,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 28,
|
|
"step_time": 598.0856093554758
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3635.0,
|
|
"completions/mean_length": 1317.609375,
|
|
"completions/mean_terminated_length": 673.3257751464844,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 2.2795989606529474,
|
|
"epoch": 0.06575963718820861,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2470703125,
|
|
"kl": 0.0005228335512583726,
|
|
"learning_rate": 9.365079365079365e-07,
|
|
"loss": -0.07,
|
|
"num_tokens": 5859167.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.34778308868408203,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.29378482699394226,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 29,
|
|
"step_time": 647.6077925832942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3737.0,
|
|
"completions/mean_length": 1378.4921875,
|
|
"completions/mean_terminated_length": 935.5129699707031,
|
|
"completions/min_length": 75.5,
|
|
"completions/min_terminated_length": 75.5,
|
|
"entropy": 3.2670377176254988,
|
|
"epoch": 0.06802721088435375,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1787109375,
|
|
"kl": 0.0004232220730955305,
|
|
"learning_rate": 9.342403628117913e-07,
|
|
"loss": 0.0307,
|
|
"num_tokens": 6060122.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.34078919887542725,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 30,
|
|
"step_time": 605.6695272815414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3844.0,
|
|
"completions/mean_length": 1264.5,
|
|
"completions/mean_terminated_length": 967.5105590820312,
|
|
"completions/min_length": 93.5,
|
|
"completions/min_terminated_length": 93.5,
|
|
"entropy": 4.282577725127339,
|
|
"epoch": 0.07029478458049887,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.0004078878860127588,
|
|
"learning_rate": 9.319727891156463e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 6247282.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.21921012550592422,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 31,
|
|
"step_time": 605.2760956226848
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3761.0,
|
|
"completions/mean_length": 1421.5625,
|
|
"completions/mean_terminated_length": 956.3410034179688,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.9570772480219603,
|
|
"epoch": 0.07256235827664399,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.220703125,
|
|
"kl": 0.00040325274949282175,
|
|
"learning_rate": 9.29705215419501e-07,
|
|
"loss": 0.0311,
|
|
"num_tokens": 6454118.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34598222374916077,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 32,
|
|
"step_time": 594.1900004739873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3791.5,
|
|
"completions/mean_length": 1409.953125,
|
|
"completions/mean_terminated_length": 912.5370483398438,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 3.804439751431346,
|
|
"epoch": 0.07482993197278912,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1376953125,
|
|
"kl": 0.00041808080868577235,
|
|
"learning_rate": 9.274376417233559e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 6657528.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.19257627427577972,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 33,
|
|
"step_time": 597.116535522975
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3800.5,
|
|
"completions/mean_length": 1313.875,
|
|
"completions/mean_terminated_length": 999.6913146972656,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.7660043127834797,
|
|
"epoch": 0.07709750566893424,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2314453125,
|
|
"kl": 0.00040203000435212743,
|
|
"learning_rate": 9.251700680272108e-07,
|
|
"loss": 0.0176,
|
|
"num_tokens": 6848420.0,
|
|
"reward": 0.09765625,
|
|
"reward_std": 0.39702552556991577,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 34,
|
|
"step_time": 588.760639490094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3577.0,
|
|
"completions/mean_length": 1154.796875,
|
|
"completions/mean_terminated_length": 673.73779296875,
|
|
"completions/min_length": 32.0,
|
|
"completions/min_terminated_length": 32.0,
|
|
"entropy": 2.520525285974145,
|
|
"epoch": 0.07936507936507936,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2490234375,
|
|
"kl": 0.00047929613219821476,
|
|
"learning_rate": 9.229024943310657e-07,
|
|
"loss": 0.0117,
|
|
"num_tokens": 7020838.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.30299752950668335,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.27048972249031067,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 35,
|
|
"step_time": 610.717342778109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3509.5,
|
|
"completions/mean_length": 1125.5546875,
|
|
"completions/mean_terminated_length": 701.979736328125,
|
|
"completions/min_length": 51.5,
|
|
"completions/min_terminated_length": 51.5,
|
|
"entropy": 3.0911447647958994,
|
|
"epoch": 0.08163265306122448,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.00045987328167029773,
|
|
"learning_rate": 9.206349206349205e-07,
|
|
"loss": 0.0551,
|
|
"num_tokens": 7187557.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3954896926879883,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 36,
|
|
"step_time": 588.8801189982332
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3893.0,
|
|
"completions/mean_length": 1400.6171875,
|
|
"completions/mean_terminated_length": 980.9603576660156,
|
|
"completions/min_length": 6.5,
|
|
"completions/min_terminated_length": 6.5,
|
|
"entropy": 3.7993264254182577,
|
|
"epoch": 0.08390022675736962,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.0004099325874449278,
|
|
"learning_rate": 9.183673469387755e-07,
|
|
"loss": 0.0228,
|
|
"num_tokens": 7391876.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.21423126757144928,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 37,
|
|
"step_time": 604.1444621784613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3806.0,
|
|
"completions/mean_length": 1468.0390625,
|
|
"completions/mean_terminated_length": 923.6571655273438,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.549352679401636,
|
|
"epoch": 0.08616780045351474,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.00044676465176962665,
|
|
"learning_rate": 9.160997732426303e-07,
|
|
"loss": 0.0222,
|
|
"num_tokens": 7603229.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.3723399490118027,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 38,
|
|
"step_time": 590.1167810624465
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3608.5,
|
|
"completions/mean_length": 1329.921875,
|
|
"completions/mean_terminated_length": 794.9758911132812,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 3.4886126536875963,
|
|
"epoch": 0.08843537414965986,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.00046471107089018915,
|
|
"learning_rate": 9.138321995464853e-07,
|
|
"loss": -0.0098,
|
|
"num_tokens": 7799027.0,
|
|
"reward": 0.14453125,
|
|
"reward_std": 0.4880780875682831,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48353175818920135,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 39,
|
|
"step_time": 606.6144362050109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3697.5,
|
|
"completions/mean_length": 1280.03125,
|
|
"completions/mean_terminated_length": 839.4837646484375,
|
|
"completions/min_length": 20.5,
|
|
"completions/min_terminated_length": 20.5,
|
|
"entropy": 3.654280535876751,
|
|
"epoch": 0.09070294784580499,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.12353515625,
|
|
"kl": 0.00046478183367071324,
|
|
"learning_rate": 9.115646258503401e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 7985183.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.3003681004047394,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 40,
|
|
"step_time": 589.6266784924082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2835.5,
|
|
"completions/mean_length": 1271.2890625,
|
|
"completions/mean_terminated_length": 773.3348693847656,
|
|
"completions/min_length": 85.5,
|
|
"completions/min_terminated_length": 85.5,
|
|
"entropy": 2.9861853010952473,
|
|
"epoch": 0.09297052154195011,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2265625,
|
|
"kl": 0.00046541515530407196,
|
|
"learning_rate": 9.09297052154195e-07,
|
|
"loss": 0.0296,
|
|
"num_tokens": 8172404.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.3428210914134979,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 41,
|
|
"step_time": 592.6969041889533
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3665.5,
|
|
"completions/mean_length": 1196.203125,
|
|
"completions/mean_terminated_length": 950.4576416015625,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 3.5352232959121466,
|
|
"epoch": 0.09523809523809523,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.00044147984135634033,
|
|
"learning_rate": 9.070294784580499e-07,
|
|
"loss": 0.0382,
|
|
"num_tokens": 8348714.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4056457281112671,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 42,
|
|
"step_time": 599.7052725390531
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3302.5,
|
|
"completions/mean_length": 1222.3828125,
|
|
"completions/mean_terminated_length": 783.2756958007812,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.064804270863533,
|
|
"epoch": 0.09750566893424037,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.251953125,
|
|
"kl": 0.0004974609882992809,
|
|
"learning_rate": 9.047619047619047e-07,
|
|
"loss": -0.0257,
|
|
"num_tokens": 8529679.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31388580799102783,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 43,
|
|
"step_time": 605.891956679523
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3939.0,
|
|
"completions/mean_length": 1661.859375,
|
|
"completions/mean_terminated_length": 1100.1346435546875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.5304269678890705,
|
|
"epoch": 0.09977324263038549,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.0004575031762215076,
|
|
"learning_rate": 9.024943310657595e-07,
|
|
"loss": 0.0182,
|
|
"num_tokens": 8766585.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.4579998552799225,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 44,
|
|
"step_time": 609.272553737741
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3914.0,
|
|
"completions/mean_length": 1193.4296875,
|
|
"completions/mean_terminated_length": 894.8565368652344,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.180556582286954,
|
|
"epoch": 0.10204081632653061,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1044921875,
|
|
"kl": 0.0004615836469383794,
|
|
"learning_rate": 9.002267573696145e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 8943480.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.21921012550592422,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 45,
|
|
"step_time": 606.0064869229682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3297.0,
|
|
"completions/mean_length": 1318.296875,
|
|
"completions/mean_terminated_length": 836.4284362792969,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 3.324997106567025,
|
|
"epoch": 0.10430839002267574,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.171875,
|
|
"kl": 0.0004851645394410298,
|
|
"learning_rate": 8.979591836734693e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 9135110.0,
|
|
"reward": 0.01171875,
|
|
"reward_std": 0.07509202510118484,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 46,
|
|
"step_time": 591.3534867856652
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3921.5,
|
|
"completions/mean_length": 1362.703125,
|
|
"completions/mean_terminated_length": 887.37744140625,
|
|
"completions/min_length": 48.0,
|
|
"completions/min_terminated_length": 48.0,
|
|
"entropy": 3.4120216239243746,
|
|
"epoch": 0.10657596371882086,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.000441622027210542,
|
|
"learning_rate": 8.956916099773242e-07,
|
|
"loss": -0.0113,
|
|
"num_tokens": 9332408.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.3428210914134979,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 47,
|
|
"step_time": 592.4939067498781
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3773.5,
|
|
"completions/mean_length": 1337.7265625,
|
|
"completions/mean_terminated_length": 889.4365234375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.588590629398823,
|
|
"epoch": 0.10884353741496598,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1806640625,
|
|
"kl": 0.0004888545086032536,
|
|
"learning_rate": 8.934240362811791e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 9528173.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.3428210914134979,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 48,
|
|
"step_time": 595.2394825532101
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3751.5,
|
|
"completions/mean_length": 1370.3125,
|
|
"completions/mean_terminated_length": 981.296630859375,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 3.3204429261386395,
|
|
"epoch": 0.1111111111111111,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1201171875,
|
|
"kl": 0.0004708280312115676,
|
|
"learning_rate": 8.91156462585034e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 9727061.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08451050519943237,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 49,
|
|
"step_time": 619.0451905168593
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3857.0,
|
|
"completions/mean_length": 1159.9296875,
|
|
"completions/mean_terminated_length": 856.0658874511719,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.274019964039326,
|
|
"epoch": 0.11337868480725624,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1904296875,
|
|
"kl": 0.00045126326358513325,
|
|
"learning_rate": 8.888888888888888e-07,
|
|
"loss": 0.025,
|
|
"num_tokens": 9899344.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.26565735042095184,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 50,
|
|
"step_time": 594.9196514263749
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3712.0,
|
|
"completions/mean_length": 1539.75,
|
|
"completions/mean_terminated_length": 1009.2075805664062,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.347749046981335,
|
|
"epoch": 0.11564625850340136,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.212890625,
|
|
"kl": 0.0004338094959166483,
|
|
"learning_rate": 8.866213151927438e-07,
|
|
"loss": 0.0298,
|
|
"num_tokens": 10119776.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31276239454746246,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 51,
|
|
"step_time": 604.532951598987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2861.5,
|
|
"completions/mean_length": 1311.625,
|
|
"completions/mean_terminated_length": 793.7900390625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.2434661109000444,
|
|
"epoch": 0.11791383219954649,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.0004534392055575154,
|
|
"learning_rate": 8.843537414965985e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 10311152.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31276239454746246,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 52,
|
|
"step_time": 606.0263563157059
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3860.0,
|
|
"completions/mean_length": 1324.3984375,
|
|
"completions/mean_terminated_length": 869.1448669433594,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 3.9168042186647654,
|
|
"epoch": 0.12018140589569161,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1103515625,
|
|
"kl": 0.0004939305722473364,
|
|
"learning_rate": 8.820861678004535e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 10502727.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.16884202510118484,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 53,
|
|
"step_time": 588.4414443285204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3656.5,
|
|
"completions/mean_length": 1435.96875,
|
|
"completions/mean_terminated_length": 885.0552673339844,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.6197192538529634,
|
|
"epoch": 0.12244897959183673,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.201171875,
|
|
"kl": 0.0004839639314013766,
|
|
"learning_rate": 8.798185941043083e-07,
|
|
"loss": -0.054,
|
|
"num_tokens": 10712659.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.3572239279747009,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 54,
|
|
"step_time": 610.2372101433575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3640.5,
|
|
"completions/mean_length": 1367.390625,
|
|
"completions/mean_terminated_length": 960.486328125,
|
|
"completions/min_length": 58.5,
|
|
"completions/min_terminated_length": 58.5,
|
|
"entropy": 3.9621335081756115,
|
|
"epoch": 0.12471655328798185,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.19921875,
|
|
"kl": 0.00046764538933530275,
|
|
"learning_rate": 8.775510204081632e-07,
|
|
"loss": 0.0121,
|
|
"num_tokens": 10911001.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.18162544071674347,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 55,
|
|
"step_time": 596.5860891141929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3507.5,
|
|
"completions/mean_length": 1238.6640625,
|
|
"completions/mean_terminated_length": 770.5470275878906,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 2.911743665114045,
|
|
"epoch": 0.12698412698412698,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.0005278955277390196,
|
|
"learning_rate": 8.752834467120181e-07,
|
|
"loss": 0.0388,
|
|
"num_tokens": 11093198.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.3057980537414551,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 56,
|
|
"step_time": 602.7431881749071
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3881.5,
|
|
"completions/mean_length": 1367.5234375,
|
|
"completions/mean_terminated_length": 978.820556640625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.401599169708788,
|
|
"epoch": 0.1292517006802721,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.0004408286617945123,
|
|
"learning_rate": 8.73015873015873e-07,
|
|
"loss": 0.029,
|
|
"num_tokens": 11293209.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5637782216072083,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.527618482708931,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.06099375709891319,
|
|
"step": 57,
|
|
"step_time": 629.6106900847517
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3375.0,
|
|
"completions/mean_length": 1267.1171875,
|
|
"completions/mean_terminated_length": 834.4556884765625,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 2.8903925102204084,
|
|
"epoch": 0.13151927437641722,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1494140625,
|
|
"kl": 0.0004891294333901897,
|
|
"learning_rate": 8.707482993197278e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 11477936.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 58,
|
|
"step_time": 591.5403750715777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3903.5,
|
|
"completions/mean_length": 1221.6484375,
|
|
"completions/mean_terminated_length": 867.8571472167969,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.4592435099184513,
|
|
"epoch": 0.13378684807256236,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.19921875,
|
|
"kl": 0.00048117998039742815,
|
|
"learning_rate": 8.684807256235828e-07,
|
|
"loss": -0.0208,
|
|
"num_tokens": 11662035.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3089591860771179,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 59,
|
|
"step_time": 617.3197298264131
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3738.0,
|
|
"completions/mean_length": 1492.890625,
|
|
"completions/mean_terminated_length": 1009.9540405273438,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.7502216193825006,
|
|
"epoch": 0.1360544217687075,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.0004347923636487394,
|
|
"learning_rate": 8.662131519274376e-07,
|
|
"loss": 0.0369,
|
|
"num_tokens": 11875833.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 60,
|
|
"step_time": 593.7963156336918
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3947.5,
|
|
"completions/mean_length": 1247.8125,
|
|
"completions/mean_terminated_length": 948.7422790527344,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.10028101503849,
|
|
"epoch": 0.1383219954648526,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2001953125,
|
|
"kl": 0.0005185582331250771,
|
|
"learning_rate": 8.639455782312925e-07,
|
|
"loss": 0.0383,
|
|
"num_tokens": 12058597.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34415413439273834,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 61,
|
|
"step_time": 595.987687363755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3756.5,
|
|
"completions/mean_length": 1450.0625,
|
|
"completions/mean_terminated_length": 811.7161560058594,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 3.791416520252824,
|
|
"epoch": 0.14058956916099774,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.216796875,
|
|
"kl": 0.000482378878587042,
|
|
"learning_rate": 8.616780045351474e-07,
|
|
"loss": 0.0598,
|
|
"num_tokens": 12270933.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.29821833968162537,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 62,
|
|
"step_time": 639.7064624852501
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3913.0,
|
|
"completions/mean_length": 1566.1171875,
|
|
"completions/mean_terminated_length": 1041.0471801757812,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.66802628710866,
|
|
"epoch": 0.14285714285714285,
|
|
"frac_reward_zero_std": 0.96875,
|
|
"grad_norm": 0.06103515625,
|
|
"kl": 0.00040643151942276745,
|
|
"learning_rate": 8.594104308390022e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 12495768.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.17536810040473938,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 63,
|
|
"step_time": 597.4973760107532
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3819.0,
|
|
"completions/mean_length": 1186.453125,
|
|
"completions/mean_terminated_length": 799.9868774414062,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 3.6843310575932264,
|
|
"epoch": 0.14512471655328799,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.0004871052587986924,
|
|
"learning_rate": 8.57142857142857e-07,
|
|
"loss": 0.0321,
|
|
"num_tokens": 12671826.0,
|
|
"reward": 0.12109375,
|
|
"reward_std": 0.4517260193824768,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.44585782289505005,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 64,
|
|
"step_time": 607.9777935571037
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3789.5,
|
|
"completions/mean_length": 1403.6953125,
|
|
"completions/mean_terminated_length": 992.7754821777344,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 4.1216657515615225,
|
|
"epoch": 0.1473922902494331,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2177734375,
|
|
"kl": 0.0004439708786776464,
|
|
"learning_rate": 8.54875283446712e-07,
|
|
"loss": 0.0501,
|
|
"num_tokens": 12874711.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3109140843153,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 65,
|
|
"step_time": 610.7431227061898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3714.0,
|
|
"completions/mean_length": 1354.2421875,
|
|
"completions/mean_terminated_length": 876.5531005859375,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"entropy": 3.687948001548648,
|
|
"epoch": 0.14965986394557823,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2080078125,
|
|
"kl": 0.0004651461445064342,
|
|
"learning_rate": 8.526077097505668e-07,
|
|
"loss": 0.0343,
|
|
"num_tokens": 13072098.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.2660238593816757,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 66,
|
|
"step_time": 593.1527040861547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4051.5,
|
|
"completions/mean_length": 1323.0625,
|
|
"completions/mean_terminated_length": 873.1241455078125,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"entropy": 4.224043572321534,
|
|
"epoch": 0.15192743764172337,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.193359375,
|
|
"kl": 0.0005088694788355497,
|
|
"learning_rate": 8.503401360544217e-07,
|
|
"loss": -0.0178,
|
|
"num_tokens": 13266110.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.3517485707998276,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 67,
|
|
"step_time": 604.5064916228876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3337.5,
|
|
"completions/mean_length": 1203.515625,
|
|
"completions/mean_terminated_length": 848.2982482910156,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.2738924492150545,
|
|
"epoch": 0.15419501133786848,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.287109375,
|
|
"kl": 0.0005158751378075976,
|
|
"learning_rate": 8.480725623582766e-07,
|
|
"loss": 0.0422,
|
|
"num_tokens": 13444504.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.4579998552799225,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 68,
|
|
"step_time": 602.0630125850439
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3615.5,
|
|
"completions/mean_length": 1392.4921875,
|
|
"completions/mean_terminated_length": 976.9803771972656,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 4.119066654704511,
|
|
"epoch": 0.1564625850340136,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.00045541685312855407,
|
|
"learning_rate": 8.458049886621315e-07,
|
|
"loss": -0.0348,
|
|
"num_tokens": 13645543.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.29378482699394226,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 69,
|
|
"step_time": 593.0577980796807
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3935.0,
|
|
"completions/mean_length": 1399.390625,
|
|
"completions/mean_terminated_length": 933.7881469726562,
|
|
"completions/min_length": 129.5,
|
|
"completions/min_terminated_length": 129.5,
|
|
"entropy": 3.355590634047985,
|
|
"epoch": 0.15873015873015872,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1806640625,
|
|
"kl": 0.0005192129283386748,
|
|
"learning_rate": 8.435374149659864e-07,
|
|
"loss": 0.0147,
|
|
"num_tokens": 13848217.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.21248632669448853,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 70,
|
|
"step_time": 599.6777586075477
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3537.0,
|
|
"completions/mean_length": 1360.4765625,
|
|
"completions/mean_terminated_length": 853.8981628417969,
|
|
"completions/min_length": 9.5,
|
|
"completions/min_terminated_length": 9.5,
|
|
"entropy": 3.191288085654378,
|
|
"epoch": 0.16099773242630386,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1748046875,
|
|
"kl": 0.0005019806021664408,
|
|
"learning_rate": 8.412698412698413e-07,
|
|
"loss": -0.0037,
|
|
"num_tokens": 14046134.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34415413439273834,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 71,
|
|
"step_time": 599.8396325754002
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3744.0,
|
|
"completions/mean_length": 1680.890625,
|
|
"completions/mean_terminated_length": 1004.6346435546875,
|
|
"completions/min_length": 50.0,
|
|
"completions/min_terminated_length": 50.0,
|
|
"entropy": 4.126177066937089,
|
|
"epoch": 0.16326530612244897,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1513671875,
|
|
"kl": 0.0005027235952184128,
|
|
"learning_rate": 8.39002267573696e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 14285372.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3408081382513046,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 72,
|
|
"step_time": 594.7445488786325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3948.5,
|
|
"completions/mean_length": 1401.875,
|
|
"completions/mean_terminated_length": 934.7810363769531,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 4.1191865261644125,
|
|
"epoch": 0.1655328798185941,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.115234375,
|
|
"kl": 0.0004872900472037145,
|
|
"learning_rate": 8.36734693877551e-07,
|
|
"loss": 0.0228,
|
|
"num_tokens": 14490792.0,
|
|
"reward": 0.02734375,
|
|
"reward_std": 0.19010721147060394,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 73,
|
|
"step_time": 605.806323479861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3813.0,
|
|
"completions/mean_length": 1314.6796875,
|
|
"completions/mean_terminated_length": 859.5545349121094,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.5274351220577955,
|
|
"epoch": 0.16780045351473924,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.232421875,
|
|
"kl": 0.0005006528513149533,
|
|
"learning_rate": 8.344671201814058e-07,
|
|
"loss": 0.0303,
|
|
"num_tokens": 14684063.0,
|
|
"reward": 0.18359375,
|
|
"reward_std": 0.5659296214580536,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.527618482708931,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 74,
|
|
"step_time": 601.1992794224061
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4038.0,
|
|
"completions/mean_length": 1398.1875,
|
|
"completions/mean_terminated_length": 953.0533142089844,
|
|
"completions/min_length": 15.5,
|
|
"completions/min_terminated_length": 15.5,
|
|
"entropy": 3.780844986438751,
|
|
"epoch": 0.17006802721088435,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.0005055789238213038,
|
|
"learning_rate": 8.321995464852607e-07,
|
|
"loss": 0.0167,
|
|
"num_tokens": 14887503.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.3414069563150406,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 75,
|
|
"step_time": 594.4315239707939
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3696.5,
|
|
"completions/mean_length": 1401.9609375,
|
|
"completions/mean_terminated_length": 932.441650390625,
|
|
"completions/min_length": 96.5,
|
|
"completions/min_terminated_length": 96.5,
|
|
"entropy": 4.313092632219195,
|
|
"epoch": 0.17233560090702948,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.23046875,
|
|
"kl": 0.0005012169604015071,
|
|
"learning_rate": 8.299319727891156e-07,
|
|
"loss": 0.0375,
|
|
"num_tokens": 15089758.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3109140843153,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 76,
|
|
"step_time": 590.3546146731824
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3951.5,
|
|
"completions/mean_length": 1343.8671875,
|
|
"completions/mean_terminated_length": 1005.885986328125,
|
|
"completions/min_length": 20.5,
|
|
"completions/min_terminated_length": 20.5,
|
|
"entropy": 3.6281288098543882,
|
|
"epoch": 0.1746031746031746,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.275390625,
|
|
"kl": 0.00048217516268778127,
|
|
"learning_rate": 8.276643990929705e-07,
|
|
"loss": 0.0711,
|
|
"num_tokens": 15289129.0,
|
|
"reward": 0.15234375,
|
|
"reward_std": 0.4942307323217392,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 77,
|
|
"step_time": 629.5171679407358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3971.5,
|
|
"completions/mean_length": 1422.21875,
|
|
"completions/mean_terminated_length": 1040.2500610351562,
|
|
"completions/min_length": 63.5,
|
|
"completions/min_terminated_length": 63.5,
|
|
"entropy": 4.338170548900962,
|
|
"epoch": 0.17687074829931973,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2041015625,
|
|
"kl": 0.0004632392774510663,
|
|
"learning_rate": 8.253968253968253e-07,
|
|
"loss": 0.0613,
|
|
"num_tokens": 15494701.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42883121967315674,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 78,
|
|
"step_time": 593.3397142631002
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3159.0,
|
|
"completions/mean_length": 1178.703125,
|
|
"completions/mean_terminated_length": 819.6268615722656,
|
|
"completions/min_length": 82.5,
|
|
"completions/min_terminated_length": 82.5,
|
|
"entropy": 3.7087379712611437,
|
|
"epoch": 0.17913832199546487,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.169921875,
|
|
"kl": 0.0005126559735799674,
|
|
"learning_rate": 8.231292517006803e-07,
|
|
"loss": 0.0204,
|
|
"num_tokens": 15669327.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.25688404589891434,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 79,
|
|
"step_time": 595.9942190372385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4061.0,
|
|
"completions/mean_length": 1321.203125,
|
|
"completions/mean_terminated_length": 924.8036193847656,
|
|
"completions/min_length": 81.0,
|
|
"completions/min_terminated_length": 81.0,
|
|
"entropy": 4.271406231448054,
|
|
"epoch": 0.18140589569160998,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.00046873624796717195,
|
|
"learning_rate": 8.208616780045351e-07,
|
|
"loss": 0.0269,
|
|
"num_tokens": 15861613.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.20661810040473938,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 80,
|
|
"step_time": 592.3808848424815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3996.5,
|
|
"completions/mean_length": 1224.046875,
|
|
"completions/mean_terminated_length": 953.5816040039062,
|
|
"completions/min_length": 96.0,
|
|
"completions/min_terminated_length": 96.0,
|
|
"entropy": 3.661328922957182,
|
|
"epoch": 0.1836734693877551,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.00046417086150540854,
|
|
"learning_rate": 8.1859410430839e-07,
|
|
"loss": 0.0969,
|
|
"num_tokens": 16040943.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5218297988176346,
|
|
"rewards/accuracy_reward_func/mean": 0.140625,
|
|
"rewards/accuracy_reward_func/std": 0.506826862692833,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 81,
|
|
"step_time": 591.166427501943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3809.5,
|
|
"completions/mean_length": 1466.0546875,
|
|
"completions/mean_terminated_length": 1007.4424133300781,
|
|
"completions/min_length": 47.5,
|
|
"completions/min_terminated_length": 47.5,
|
|
"entropy": 4.402091367170215,
|
|
"epoch": 0.18594104308390022,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2099609375,
|
|
"kl": 0.0004720119945886836,
|
|
"learning_rate": 8.163265306122449e-07,
|
|
"loss": 0.0212,
|
|
"num_tokens": 16252866.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.31193090975284576,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 82,
|
|
"step_time": 601.9541582926176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3953.5,
|
|
"completions/mean_length": 1354.015625,
|
|
"completions/mean_terminated_length": 847.1114807128906,
|
|
"completions/min_length": 30.0,
|
|
"completions/min_terminated_length": 30.0,
|
|
"entropy": 3.4956730902194977,
|
|
"epoch": 0.18820861678004536,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.0005511523636414495,
|
|
"learning_rate": 8.140589569160997e-07,
|
|
"loss": -0.0169,
|
|
"num_tokens": 16450664.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.3938441723585129,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 83,
|
|
"step_time": 600.6701087602414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3605.0,
|
|
"completions/mean_length": 1505.0234375,
|
|
"completions/mean_terminated_length": 1057.6773681640625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.234385251998901,
|
|
"epoch": 0.19047619047619047,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1640625,
|
|
"kl": 0.00045168545375418034,
|
|
"learning_rate": 8.117913832199546e-07,
|
|
"loss": 0.016,
|
|
"num_tokens": 16666115.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.25652606785297394,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 84,
|
|
"step_time": 587.7375784683973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3895.5,
|
|
"completions/mean_length": 1411.1875,
|
|
"completions/mean_terminated_length": 971.8545227050781,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 3.7613945519551635,
|
|
"epoch": 0.1927437641723356,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.000517658699209278,
|
|
"learning_rate": 8.095238095238095e-07,
|
|
"loss": -0.0027,
|
|
"num_tokens": 16871639.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.30986589193344116,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 85,
|
|
"step_time": 604.7108815400861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3775.0,
|
|
"completions/mean_length": 1438.3203125,
|
|
"completions/mean_terminated_length": 1085.2908935546875,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.597478030249476,
|
|
"epoch": 0.19501133786848074,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2353515625,
|
|
"kl": 0.0004401639350817277,
|
|
"learning_rate": 8.072562358276643e-07,
|
|
"loss": 0.0232,
|
|
"num_tokens": 17079948.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 86,
|
|
"step_time": 594.5786706414074
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3547.0,
|
|
"completions/mean_length": 1463.40625,
|
|
"completions/mean_terminated_length": 885.6181335449219,
|
|
"completions/min_length": 42.5,
|
|
"completions/min_terminated_length": 42.5,
|
|
"entropy": 3.2632428109645844,
|
|
"epoch": 0.19727891156462585,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.09521484375,
|
|
"kl": 0.0005363671834857087,
|
|
"learning_rate": 8.049886621315193e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 17291036.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 87,
|
|
"step_time": 591.2662706999108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3408.0,
|
|
"completions/mean_length": 1145.75,
|
|
"completions/mean_terminated_length": 748.8887329101562,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 2.831279480829835,
|
|
"epoch": 0.19954648526077098,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.0006093931369832717,
|
|
"learning_rate": 8.027210884353741e-07,
|
|
"loss": -0.0124,
|
|
"num_tokens": 17464188.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3244084417819977,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.27048972249031067,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 88,
|
|
"step_time": 648.395480513107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3822.0,
|
|
"completions/mean_length": 1535.40625,
|
|
"completions/mean_terminated_length": 975.2217102050781,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.225502125918865,
|
|
"epoch": 0.2018140589569161,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.234375,
|
|
"kl": 0.0004882277917204192,
|
|
"learning_rate": 8.00453514739229e-07,
|
|
"loss": 0.0023,
|
|
"num_tokens": 17685020.0,
|
|
"reward": 0.10546875,
|
|
"reward_std": 0.42434585094451904,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.41934312880039215,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 89,
|
|
"step_time": 617.4666036618873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3873.5,
|
|
"completions/mean_length": 1306.28125,
|
|
"completions/mean_terminated_length": 991.5018310546875,
|
|
"completions/min_length": 69.5,
|
|
"completions/min_terminated_length": 69.5,
|
|
"entropy": 4.6000791024416685,
|
|
"epoch": 0.20408163265306123,
|
|
"frac_reward_zero_std": 0.71875,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.0004467499857128132,
|
|
"learning_rate": 7.981859410430839e-07,
|
|
"loss": 0.0685,
|
|
"num_tokens": 17874960.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6037300229072571,
|
|
"rewards/accuracy_reward_func/mean": 0.203125,
|
|
"rewards/accuracy_reward_func/std": 0.6038230657577515,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 90,
|
|
"step_time": 604.7552698785439
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4045.0,
|
|
"completions/mean_length": 1351.59375,
|
|
"completions/mean_terminated_length": 843.3703918457031,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.1051435321569443,
|
|
"epoch": 0.20634920634920634,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.189453125,
|
|
"kl": 0.0005171954862817074,
|
|
"learning_rate": 7.959183673469388e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 18072192.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.21445615589618683,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 91,
|
|
"step_time": 600.6844751848839
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4009.5,
|
|
"completions/mean_length": 1182.4375,
|
|
"completions/mean_terminated_length": 914.9233093261719,
|
|
"completions/min_length": 43.5,
|
|
"completions/min_terminated_length": 43.5,
|
|
"entropy": 3.176400328055024,
|
|
"epoch": 0.20861678004535147,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.228515625,
|
|
"kl": 0.0006316097587841796,
|
|
"learning_rate": 7.936507936507936e-07,
|
|
"loss": 0.0713,
|
|
"num_tokens": 18247568.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34415413439273834,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 92,
|
|
"step_time": 600.9037633007392
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3857.5,
|
|
"completions/mean_length": 1264.0234375,
|
|
"completions/mean_terminated_length": 885.5155334472656,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 3.3863612934947014,
|
|
"epoch": 0.2108843537414966,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1533203125,
|
|
"kl": 0.0005828815615132044,
|
|
"learning_rate": 7.913832199546485e-07,
|
|
"loss": 0.0275,
|
|
"num_tokens": 18432431.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.20661810040473938,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 93,
|
|
"step_time": 588.9628076292574
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3825.5,
|
|
"completions/mean_length": 1464.2578125,
|
|
"completions/mean_terminated_length": 920.7213745117188,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.7992858309298754,
|
|
"epoch": 0.21315192743764172,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1748046875,
|
|
"kl": 0.0004975047263542365,
|
|
"learning_rate": 7.891156462585033e-07,
|
|
"loss": 0.0563,
|
|
"num_tokens": 18642508.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.3037330359220505,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 94,
|
|
"step_time": 590.199854795821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3217.0,
|
|
"completions/mean_length": 1518.265625,
|
|
"completions/mean_terminated_length": 873.223388671875,
|
|
"completions/min_length": 34.0,
|
|
"completions/min_terminated_length": 34.0,
|
|
"entropy": 3.0404637344181538,
|
|
"epoch": 0.21541950113378686,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2001953125,
|
|
"kl": 0.0006586737342786364,
|
|
"learning_rate": 7.868480725623582e-07,
|
|
"loss": 0.0262,
|
|
"num_tokens": 18861754.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.37508824467658997,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 95,
|
|
"step_time": 600.110039126128
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3854.0,
|
|
"completions/mean_length": 1147.1640625,
|
|
"completions/mean_terminated_length": 814.8431091308594,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"entropy": 3.5139937046915293,
|
|
"epoch": 0.21768707482993196,
|
|
"frac_reward_zero_std": 0.6875,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.0006208947957020428,
|
|
"learning_rate": 7.845804988662131e-07,
|
|
"loss": 0.0483,
|
|
"num_tokens": 19033075.0,
|
|
"reward": 0.18359375,
|
|
"reward_std": 0.5316187739372253,
|
|
"rewards/accuracy_reward_func/mean": 0.171875,
|
|
"rewards/accuracy_reward_func/std": 0.5257505476474762,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 96,
|
|
"step_time": 612.0829366515391
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3766.0,
|
|
"completions/mean_length": 1298.578125,
|
|
"completions/mean_terminated_length": 937.0536499023438,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 4.03733023442328,
|
|
"epoch": 0.2199546485260771,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1826171875,
|
|
"kl": 0.0005434328236333386,
|
|
"learning_rate": 7.82312925170068e-07,
|
|
"loss": 0.0332,
|
|
"num_tokens": 19225345.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.2878170385956764,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 97,
|
|
"step_time": 620.4109515491873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3946.5,
|
|
"completions/mean_length": 1291.34375,
|
|
"completions/mean_terminated_length": 1162.2857360839844,
|
|
"completions/min_length": 64.5,
|
|
"completions/min_terminated_length": 64.5,
|
|
"entropy": 4.210914688184857,
|
|
"epoch": 0.2222222222222222,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.36328125,
|
|
"kl": 0.0004791390597347345,
|
|
"learning_rate": 7.800453514739229e-07,
|
|
"loss": -0.0068,
|
|
"num_tokens": 19414265.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34598222374916077,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 98,
|
|
"step_time": 587.3077208646573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3505.5,
|
|
"completions/mean_length": 1350.390625,
|
|
"completions/mean_terminated_length": 900.6047668457031,
|
|
"completions/min_length": 147.5,
|
|
"completions/min_terminated_length": 147.5,
|
|
"entropy": 3.5034069549292326,
|
|
"epoch": 0.22448979591836735,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.240234375,
|
|
"kl": 0.0005506929642251635,
|
|
"learning_rate": 7.777777777777778e-07,
|
|
"loss": -0.0211,
|
|
"num_tokens": 19610999.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.3185287415981293,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 99,
|
|
"step_time": 600.8466322016902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4064.0,
|
|
"completions/mean_length": 1142.0703125,
|
|
"completions/mean_terminated_length": 867.5115661621094,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 3.881171302869916,
|
|
"epoch": 0.22675736961451248,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.205078125,
|
|
"kl": 0.0005395735774982313,
|
|
"learning_rate": 7.755102040816326e-07,
|
|
"loss": 0.011,
|
|
"num_tokens": 19779508.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.20516829937696457,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 100,
|
|
"step_time": 591.9800079758279
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3986.5,
|
|
"completions/mean_length": 1238.7421875,
|
|
"completions/mean_terminated_length": 945.7524108886719,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.6473589222878218,
|
|
"epoch": 0.2290249433106576,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.0005329903901838406,
|
|
"learning_rate": 7.732426303854876e-07,
|
|
"loss": 0.0061,
|
|
"num_tokens": 19963339.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.31631264090538025,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 101,
|
|
"step_time": 604.477686543949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3731.5,
|
|
"completions/mean_length": 1222.5078125,
|
|
"completions/mean_terminated_length": 839.1725769042969,
|
|
"completions/min_length": 63.5,
|
|
"completions/min_terminated_length": 63.5,
|
|
"entropy": 3.7527093160897493,
|
|
"epoch": 0.23129251700680273,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2314453125,
|
|
"kl": 0.0005475037442010944,
|
|
"learning_rate": 7.709750566893424e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 20143212.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.22127514332532883,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 102,
|
|
"step_time": 592.505566266831
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4022.0,
|
|
"completions/mean_length": 1107.6328125,
|
|
"completions/mean_terminated_length": 798.4913635253906,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.091963429003954,
|
|
"epoch": 0.23356009070294784,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.0005909597002755618,
|
|
"learning_rate": 7.687074829931972e-07,
|
|
"loss": -0.0608,
|
|
"num_tokens": 20309593.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4511355459690094,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.44585782289505005,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 103,
|
|
"step_time": 615.5466353255324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3392.5,
|
|
"completions/mean_length": 1189.453125,
|
|
"completions/mean_terminated_length": 710.3790283203125,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 2.8702713698148727,
|
|
"epoch": 0.23582766439909297,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.146484375,
|
|
"kl": 0.0005800623353025003,
|
|
"learning_rate": 7.664399092970521e-07,
|
|
"loss": 0.0195,
|
|
"num_tokens": 20485631.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.4352022409439087,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.06099375709891319,
|
|
"step": 104,
|
|
"step_time": 596.6378222648054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3404.5,
|
|
"completions/mean_length": 1230.7265625,
|
|
"completions/mean_terminated_length": 762.2291870117188,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 3.1164721120148897,
|
|
"epoch": 0.23809523809523808,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.2294921875,
|
|
"kl": 0.0006275299128901679,
|
|
"learning_rate": 7.64172335600907e-07,
|
|
"loss": 0.0481,
|
|
"num_tokens": 20667916.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.16574779152870178,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 105,
|
|
"step_time": 595.2047035517171
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3978.0,
|
|
"completions/mean_length": 1424.9453125,
|
|
"completions/mean_terminated_length": 901.153564453125,
|
|
"completions/min_length": 44.5,
|
|
"completions/min_terminated_length": 44.5,
|
|
"entropy": 3.49599988758564,
|
|
"epoch": 0.24036281179138322,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.11865234375,
|
|
"kl": 0.0006155359710646735,
|
|
"learning_rate": 7.619047619047618e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 20873273.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08451050519943237,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 106,
|
|
"step_time": 587.4538793694228
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3708.5,
|
|
"completions/mean_length": 1262.1484375,
|
|
"completions/mean_terminated_length": 885.9746398925781,
|
|
"completions/min_length": 19.0,
|
|
"completions/min_terminated_length": 19.0,
|
|
"entropy": 3.566706972196698,
|
|
"epoch": 0.24263038548752835,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.169921875,
|
|
"kl": 0.0005286504069772491,
|
|
"learning_rate": 7.596371882086168e-07,
|
|
"loss": 0.0078,
|
|
"num_tokens": 21060904.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.41608040034770966,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 107,
|
|
"step_time": 621.596909858752
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3543.0,
|
|
"completions/mean_length": 1123.296875,
|
|
"completions/mean_terminated_length": 787.1844787597656,
|
|
"completions/min_length": 8.5,
|
|
"completions/min_terminated_length": 8.5,
|
|
"entropy": 3.004608254879713,
|
|
"epoch": 0.24489795918367346,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.0006105130787545932,
|
|
"learning_rate": 7.573696145124716e-07,
|
|
"loss": 0.0024,
|
|
"num_tokens": 21227694.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.3587139919400215,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.3145764470100403,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 108,
|
|
"step_time": 590.9640978942625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3571.0,
|
|
"completions/mean_length": 1324.8046875,
|
|
"completions/mean_terminated_length": 837.8250732421875,
|
|
"completions/min_length": 26.0,
|
|
"completions/min_terminated_length": 26.0,
|
|
"entropy": 3.4616124592721462,
|
|
"epoch": 0.2471655328798186,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2275390625,
|
|
"kl": 0.0006760806036254507,
|
|
"learning_rate": 7.551020408163265e-07,
|
|
"loss": -0.0509,
|
|
"num_tokens": 21423821.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.4644871801137924,
|
|
"rewards/accuracy_reward_func/mean": 0.140625,
|
|
"rewards/accuracy_reward_func/std": 0.4583333432674408,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 109,
|
|
"step_time": 641.2328396155499
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3927.0,
|
|
"completions/mean_length": 1127.7734375,
|
|
"completions/mean_terminated_length": 855.1835327148438,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.090597476810217,
|
|
"epoch": 0.2494331065759637,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.0005724334750993876,
|
|
"learning_rate": 7.528344671201814e-07,
|
|
"loss": 0.0169,
|
|
"num_tokens": 21591224.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.42214976251125336,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.41878482699394226,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 110,
|
|
"step_time": 599.4026822941378
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3730.0,
|
|
"completions/mean_length": 1296.1875,
|
|
"completions/mean_terminated_length": 839.9024658203125,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 3.567176640033722,
|
|
"epoch": 0.25170068027210885,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.154296875,
|
|
"kl": 0.0006291964350566559,
|
|
"learning_rate": 7.505668934240363e-07,
|
|
"loss": 0.0045,
|
|
"num_tokens": 21781100.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.25046011805534363,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 111,
|
|
"step_time": 610.2894422458485
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3927.0,
|
|
"completions/mean_length": 1353.1953125,
|
|
"completions/mean_terminated_length": 989.6641845703125,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 3.818270405754447,
|
|
"epoch": 0.25396825396825395,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.271484375,
|
|
"kl": 0.0005479211945385032,
|
|
"learning_rate": 7.482993197278912e-07,
|
|
"loss": 0.0266,
|
|
"num_tokens": 21979421.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5287948995828629,
|
|
"rewards/accuracy_reward_func/mean": 0.140625,
|
|
"rewards/accuracy_reward_func/std": 0.506826862692833,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 112,
|
|
"step_time": 620.8200170784257
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3562.5,
|
|
"completions/mean_length": 1262.40625,
|
|
"completions/mean_terminated_length": 939.8333740234375,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 3.4629177935421467,
|
|
"epoch": 0.2562358276643991,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.271484375,
|
|
"kl": 0.000551530971279135,
|
|
"learning_rate": 7.46031746031746e-07,
|
|
"loss": 0.0681,
|
|
"num_tokens": 22165293.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.369957834482193,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 113,
|
|
"step_time": 603.7505081258714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3820.5,
|
|
"completions/mean_length": 1239.3515625,
|
|
"completions/mean_terminated_length": 830.1184387207031,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 3.3482427280396223,
|
|
"epoch": 0.2585034013605442,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.24609375,
|
|
"kl": 0.000703619014529977,
|
|
"learning_rate": 7.437641723356008e-07,
|
|
"loss": 0.0205,
|
|
"num_tokens": 22346438.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.4629179835319519,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 114,
|
|
"step_time": 586.9423351995647
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3797.5,
|
|
"completions/mean_length": 1386.4453125,
|
|
"completions/mean_terminated_length": 1086.4455261230469,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.8636064995080233,
|
|
"epoch": 0.26077097505668934,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.0005316978449627641,
|
|
"learning_rate": 7.414965986394558e-07,
|
|
"loss": -0.0043,
|
|
"num_tokens": 22549879.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.31661439687013626,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.27048972249031067,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 115,
|
|
"step_time": 624.9861192265525
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3938.0,
|
|
"completions/mean_length": 1533.71875,
|
|
"completions/mean_terminated_length": 970.8081665039062,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.5712718572467566,
|
|
"epoch": 0.26303854875283444,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2294921875,
|
|
"kl": 0.0005737728936310305,
|
|
"learning_rate": 7.392290249433106e-07,
|
|
"loss": 0.0217,
|
|
"num_tokens": 22768591.0,
|
|
"reward": 0.12109375,
|
|
"reward_std": 0.4603298455476761,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 116,
|
|
"step_time": 593.1672237035818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4057.5,
|
|
"completions/mean_length": 1316.34375,
|
|
"completions/mean_terminated_length": 830.1437683105469,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 4.00539525039494,
|
|
"epoch": 0.2653061224489796,
|
|
"frac_reward_zero_std": 0.71875,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.0006508865458272339,
|
|
"learning_rate": 7.369614512471655e-07,
|
|
"loss": 0.0692,
|
|
"num_tokens": 22963011.0,
|
|
"reward": 0.17578125,
|
|
"reward_std": 0.5568371117115021,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.537759855389595,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 117,
|
|
"step_time": 602.8105560471304
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3727.0,
|
|
"completions/mean_length": 1190.484375,
|
|
"completions/mean_terminated_length": 775.3773498535156,
|
|
"completions/min_length": 6.5,
|
|
"completions/min_terminated_length": 6.5,
|
|
"entropy": 3.024929428473115,
|
|
"epoch": 0.2675736961451247,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.0006956955448913504,
|
|
"learning_rate": 7.346938775510204e-07,
|
|
"loss": 0.0548,
|
|
"num_tokens": 23139237.0,
|
|
"reward": 0.09765625,
|
|
"reward_std": 0.3964306712150574,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 118,
|
|
"step_time": 600.0795874646865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4051.5,
|
|
"completions/mean_length": 1415.296875,
|
|
"completions/mean_terminated_length": 924.1951599121094,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.7416524589061737,
|
|
"epoch": 0.2698412698412698,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.24609375,
|
|
"kl": 0.0006022217125973839,
|
|
"learning_rate": 7.324263038548753e-07,
|
|
"loss": -0.0157,
|
|
"num_tokens": 23345851.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.1596149355173111,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 119,
|
|
"step_time": 608.6475320761092
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3477.5,
|
|
"completions/mean_length": 1361.96875,
|
|
"completions/mean_terminated_length": 824.310791015625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.2436249125748873,
|
|
"epoch": 0.272108843537415,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1923828125,
|
|
"kl": 0.0006365020803968946,
|
|
"learning_rate": 7.301587301587301e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 23546123.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3109140843153,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 120,
|
|
"step_time": 611.0621263957582
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3484.0,
|
|
"completions/mean_length": 1260.0,
|
|
"completions/mean_terminated_length": 702.6099548339844,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 2.631860775873065,
|
|
"epoch": 0.2743764172335601,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.0007836887680241489,
|
|
"learning_rate": 7.278911564625851e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 23730211.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36722832918167114,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 121,
|
|
"step_time": 595.9544806145132
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3760.0,
|
|
"completions/mean_length": 1218.25,
|
|
"completions/mean_terminated_length": 861.7281799316406,
|
|
"completions/min_length": 34.0,
|
|
"completions/min_terminated_length": 34.0,
|
|
"entropy": 3.6289214845746756,
|
|
"epoch": 0.2766439909297052,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2080078125,
|
|
"kl": 0.0006548811434186064,
|
|
"learning_rate": 7.256235827664399e-07,
|
|
"loss": 0.0193,
|
|
"num_tokens": 23908535.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.31498996168375015,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.27048972249031067,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 122,
|
|
"step_time": 589.5355450129136
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3766.0,
|
|
"completions/mean_length": 1185.2109375,
|
|
"completions/mean_terminated_length": 798.7309265136719,
|
|
"completions/min_length": 24.5,
|
|
"completions/min_terminated_length": 24.5,
|
|
"entropy": 2.9175926027819514,
|
|
"epoch": 0.2789115646258503,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.23046875,
|
|
"kl": 0.0006433413955164724,
|
|
"learning_rate": 7.233560090702947e-07,
|
|
"loss": -0.0112,
|
|
"num_tokens": 24082610.0,
|
|
"reward": 0.12109375,
|
|
"reward_std": 0.4948337972164154,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 123,
|
|
"step_time": 596.0065838880837
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3330.5,
|
|
"completions/mean_length": 1321.828125,
|
|
"completions/mean_terminated_length": 871.0489501953125,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 3.2711939439177513,
|
|
"epoch": 0.2811791383219955,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.224609375,
|
|
"kl": 0.0006157849429655471,
|
|
"learning_rate": 7.210884353741496e-07,
|
|
"loss": -0.0196,
|
|
"num_tokens": 24276268.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3454075902700424,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 124,
|
|
"step_time": 594.5184129439294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4054.5,
|
|
"completions/mean_length": 1458.1796875,
|
|
"completions/mean_terminated_length": 970.0838623046875,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.947012532502413,
|
|
"epoch": 0.2834467120181406,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.177734375,
|
|
"kl": 0.0007149179364205338,
|
|
"learning_rate": 7.188208616780045e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 24488735.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3109140843153,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 125,
|
|
"step_time": 617.1531918440014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3989.5,
|
|
"completions/mean_length": 1127.7890625,
|
|
"completions/mean_terminated_length": 761.6597900390625,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 3.1962913321331143,
|
|
"epoch": 0.2857142857142857,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.224609375,
|
|
"kl": 0.000640524232039752,
|
|
"learning_rate": 7.165532879818594e-07,
|
|
"loss": 0.0669,
|
|
"num_tokens": 24656648.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.3414069563150406,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 126,
|
|
"step_time": 599.5053179264069
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3744.0,
|
|
"completions/mean_length": 1123.3359375,
|
|
"completions/mean_terminated_length": 871.4152526855469,
|
|
"completions/min_length": 37.0,
|
|
"completions/min_terminated_length": 37.0,
|
|
"entropy": 4.046359576284885,
|
|
"epoch": 0.28798185941043086,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.248046875,
|
|
"kl": 0.0005671191984220059,
|
|
"learning_rate": 7.142857142857143e-07,
|
|
"loss": 0.0291,
|
|
"num_tokens": 24822483.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.5039452314376831,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48353175818920135,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 127,
|
|
"step_time": 603.7964815716259
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3985.0,
|
|
"completions/mean_length": 1488.3671875,
|
|
"completions/mean_terminated_length": 1012.0201721191406,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.231710430234671,
|
|
"epoch": 0.29024943310657597,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1845703125,
|
|
"kl": 0.0006102154534346482,
|
|
"learning_rate": 7.120181405895691e-07,
|
|
"loss": -0.0175,
|
|
"num_tokens": 25036834.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.35961462557315826,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 128,
|
|
"step_time": 598.2676164745353
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3743.5,
|
|
"completions/mean_length": 1232.59375,
|
|
"completions/mean_terminated_length": 880.9473571777344,
|
|
"completions/min_length": 7.5,
|
|
"completions/min_terminated_length": 7.5,
|
|
"entropy": 3.7094720266759396,
|
|
"epoch": 0.2925170068027211,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.0006456134674408531,
|
|
"learning_rate": 7.097505668934241e-07,
|
|
"loss": 0.0118,
|
|
"num_tokens": 25220690.0,
|
|
"reward": 0.12890625,
|
|
"reward_std": 0.49247273802757263,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.44585782289505005,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 129,
|
|
"step_time": 630.2013517902233
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3993.5,
|
|
"completions/mean_length": 1402.328125,
|
|
"completions/mean_terminated_length": 971.8533630371094,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.993540972471237,
|
|
"epoch": 0.2947845804988662,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30859375,
|
|
"kl": 0.0005732664512834162,
|
|
"learning_rate": 7.074829931972789e-07,
|
|
"loss": -0.045,
|
|
"num_tokens": 25423684.0,
|
|
"reward": 0.10546875,
|
|
"reward_std": 0.4302453547716141,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 130,
|
|
"step_time": 603.3049238868989
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3701.0,
|
|
"completions/mean_length": 1241.9296875,
|
|
"completions/mean_terminated_length": 833.1251831054688,
|
|
"completions/min_length": 7.5,
|
|
"completions/min_terminated_length": 7.5,
|
|
"entropy": 3.7468552589416504,
|
|
"epoch": 0.29705215419501135,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.23046875,
|
|
"kl": 0.0006608536746171012,
|
|
"learning_rate": 7.052154195011338e-07,
|
|
"loss": -0.0096,
|
|
"num_tokens": 25607707.0,
|
|
"reward": 0.10546875,
|
|
"reward_std": 0.42330680787563324,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.41934312880039215,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 131,
|
|
"step_time": 613.6388336997479
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3875.0,
|
|
"completions/mean_length": 1451.390625,
|
|
"completions/mean_terminated_length": 952.4648742675781,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.6195167172700167,
|
|
"epoch": 0.29931972789115646,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.0006574355638804263,
|
|
"learning_rate": 7.029478458049887e-07,
|
|
"loss": 0.0558,
|
|
"num_tokens": 25819009.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.31193090975284576,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 132,
|
|
"step_time": 605.7453820910305
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3493.0,
|
|
"completions/mean_length": 1387.8046875,
|
|
"completions/mean_terminated_length": 856.1520080566406,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 2.9871059209108353,
|
|
"epoch": 0.30158730158730157,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.259765625,
|
|
"kl": 0.000651665689474612,
|
|
"learning_rate": 7.006802721088436e-07,
|
|
"loss": -0.0238,
|
|
"num_tokens": 26021612.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.35682113468647003,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 133,
|
|
"step_time": 602.8428930342197
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3892.0,
|
|
"completions/mean_length": 1331.25,
|
|
"completions/mean_terminated_length": 936.5840454101562,
|
|
"completions/min_length": 74.5,
|
|
"completions/min_terminated_length": 74.5,
|
|
"entropy": 3.4091415740549564,
|
|
"epoch": 0.30385487528344673,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.205078125,
|
|
"kl": 0.0005923464564148162,
|
|
"learning_rate": 6.984126984126983e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 26214596.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3089591860771179,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 134,
|
|
"step_time": 592.2523580477573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4025.5,
|
|
"completions/mean_length": 1326.265625,
|
|
"completions/mean_terminated_length": 814.2611083984375,
|
|
"completions/min_length": 65.5,
|
|
"completions/min_terminated_length": 65.5,
|
|
"entropy": 3.5235285479575396,
|
|
"epoch": 0.30612244897959184,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.0006541062161886657,
|
|
"learning_rate": 6.961451247165533e-07,
|
|
"loss": 0.0041,
|
|
"num_tokens": 26412046.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.2760659530758858,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 135,
|
|
"step_time": 634.69742300082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3145.0,
|
|
"completions/mean_length": 1532.4765625,
|
|
"completions/mean_terminated_length": 912.8550720214844,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.4465801026672125,
|
|
"epoch": 0.30839002267573695,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.216796875,
|
|
"kl": 0.0006568615767719166,
|
|
"learning_rate": 6.938775510204081e-07,
|
|
"loss": -0.0382,
|
|
"num_tokens": 26632051.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.3723399490118027,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 136,
|
|
"step_time": 590.2187935248949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3837.5,
|
|
"completions/mean_length": 1374.9921875,
|
|
"completions/mean_terminated_length": 1065.7265014648438,
|
|
"completions/min_length": 84.5,
|
|
"completions/min_terminated_length": 84.5,
|
|
"entropy": 4.470084089785814,
|
|
"epoch": 0.31065759637188206,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.000530708899077581,
|
|
"learning_rate": 6.91609977324263e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 26833150.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.3166804313659668,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 137,
|
|
"step_time": 610.4842649418861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3891.0,
|
|
"completions/mean_length": 1168.3828125,
|
|
"completions/mean_terminated_length": 835.8844604492188,
|
|
"completions/min_length": 25.0,
|
|
"completions/min_terminated_length": 25.0,
|
|
"entropy": 3.404021816328168,
|
|
"epoch": 0.3129251700680272,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.0007529365475420491,
|
|
"learning_rate": 6.893424036281179e-07,
|
|
"loss": 0.044,
|
|
"num_tokens": 27006863.0,
|
|
"reward": 0.13671875,
|
|
"reward_std": 0.44627492129802704,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.4395764470100403,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 138,
|
|
"step_time": 593.758818234317
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3604.5,
|
|
"completions/mean_length": 1632.1875,
|
|
"completions/mean_terminated_length": 1059.4151916503906,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 3.9975744858384132,
|
|
"epoch": 0.31519274376417233,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.158203125,
|
|
"kl": 0.0007581692070743884,
|
|
"learning_rate": 6.870748299319728e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 27239883.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.3924107551574707,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 139,
|
|
"step_time": 597.7115042014048
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3864.5,
|
|
"completions/mean_length": 1311.5234375,
|
|
"completions/mean_terminated_length": 997.4262084960938,
|
|
"completions/min_length": 20.5,
|
|
"completions/min_terminated_length": 20.5,
|
|
"entropy": 3.996813965961337,
|
|
"epoch": 0.31746031746031744,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.171875,
|
|
"kl": 0.0005961597184978018,
|
|
"learning_rate": 6.848072562358276e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 27430910.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 140,
|
|
"step_time": 593.0021959957667
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0546875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3527.5,
|
|
"completions/mean_length": 1030.7890625,
|
|
"completions/mean_terminated_length": 852.6925354003906,
|
|
"completions/min_length": 30.0,
|
|
"completions/min_terminated_length": 30.0,
|
|
"entropy": 3.5588269783183932,
|
|
"epoch": 0.3197278911564626,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.228515625,
|
|
"kl": 0.0006732991332683014,
|
|
"learning_rate": 6.825396825396826e-07,
|
|
"loss": 0.0315,
|
|
"num_tokens": 27587051.0,
|
|
"reward": 0.15234375,
|
|
"reward_std": 0.5549117624759674,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.46915292739868164,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 141,
|
|
"step_time": 622.254900438711
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3744.5,
|
|
"completions/mean_length": 1502.4453125,
|
|
"completions/mean_terminated_length": 932.8360290527344,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.243277547881007,
|
|
"epoch": 0.3219954648526077,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.15625,
|
|
"kl": 0.0007283171826202306,
|
|
"learning_rate": 6.802721088435374e-07,
|
|
"loss": 0.0269,
|
|
"num_tokens": 27804500.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.302433118224144,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 142,
|
|
"step_time": 599.0916443401948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3908.0,
|
|
"completions/mean_length": 1247.8515625,
|
|
"completions/mean_terminated_length": 925.2738952636719,
|
|
"completions/min_length": 54.0,
|
|
"completions/min_terminated_length": 54.0,
|
|
"entropy": 3.369177982211113,
|
|
"epoch": 0.3242630385487528,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.0006683723759124405,
|
|
"learning_rate": 6.780045351473924e-07,
|
|
"loss": 0.0212,
|
|
"num_tokens": 27988565.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.3428210914134979,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 143,
|
|
"step_time": 596.4260103851557
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3616.0,
|
|
"completions/mean_length": 1347.6640625,
|
|
"completions/mean_terminated_length": 900.01953125,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.636057852767408,
|
|
"epoch": 0.32653061224489793,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.173828125,
|
|
"kl": 0.0006290206483754446,
|
|
"learning_rate": 6.757369614512471e-07,
|
|
"loss": 0.0129,
|
|
"num_tokens": 28185642.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3126605153083801,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.06099375709891319,
|
|
"step": 144,
|
|
"step_time": 608.9668316529132
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3913.0,
|
|
"completions/mean_length": 1402.109375,
|
|
"completions/mean_terminated_length": 843.0,
|
|
"completions/min_length": 47.0,
|
|
"completions/min_terminated_length": 47.0,
|
|
"entropy": 3.871475676074624,
|
|
"epoch": 0.3287981859410431,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1796875,
|
|
"kl": 0.0006890235495120578,
|
|
"learning_rate": 6.734693877551019e-07,
|
|
"loss": 0.0074,
|
|
"num_tokens": 28388800.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.2582981809973717,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 145,
|
|
"step_time": 594.1406287304126
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3906.5,
|
|
"completions/mean_length": 1437.7109375,
|
|
"completions/mean_terminated_length": 945.7689514160156,
|
|
"completions/min_length": 43.0,
|
|
"completions/min_terminated_length": 43.0,
|
|
"entropy": 3.0443298555910587,
|
|
"epoch": 0.3310657596371882,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.11962890625,
|
|
"kl": 0.0009946563122866792,
|
|
"learning_rate": 6.712018140589569e-07,
|
|
"loss": 0.024,
|
|
"num_tokens": 28597619.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.18123632669448853,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 146,
|
|
"step_time": 602.034218671266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3890.5,
|
|
"completions/mean_length": 1303.3359375,
|
|
"completions/mean_terminated_length": 931.7557983398438,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.749638095498085,
|
|
"epoch": 0.3333333333333333,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.2197265625,
|
|
"kl": 0.0006567600012203911,
|
|
"learning_rate": 6.689342403628118e-07,
|
|
"loss": 0.0424,
|
|
"num_tokens": 28796006.0,
|
|
"reward": 0.02734375,
|
|
"reward_std": 0.16277606785297394,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 147,
|
|
"step_time": 699.574759578798
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3726.5,
|
|
"completions/mean_length": 1353.8671875,
|
|
"completions/mean_terminated_length": 908.179931640625,
|
|
"completions/min_length": 30.5,
|
|
"completions/min_terminated_length": 30.5,
|
|
"entropy": 4.074944546446204,
|
|
"epoch": 0.3356009070294785,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.2255859375,
|
|
"kl": 0.000654284445772646,
|
|
"learning_rate": 6.666666666666666e-07,
|
|
"loss": -0.0143,
|
|
"num_tokens": 28992309.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.3003681004047394,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 148,
|
|
"step_time": 600.8639830420725
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3804.0,
|
|
"completions/mean_length": 1365.90625,
|
|
"completions/mean_terminated_length": 860.3333435058594,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 2.8986724950373173,
|
|
"epoch": 0.3378684807256236,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2412109375,
|
|
"kl": 0.0007142787781049265,
|
|
"learning_rate": 6.643990929705216e-07,
|
|
"loss": 0.0252,
|
|
"num_tokens": 29188805.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34415413439273834,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 149,
|
|
"step_time": 588.2603411567397
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2942.0,
|
|
"completions/mean_length": 1108.125,
|
|
"completions/mean_terminated_length": 800.030029296875,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 3.307353427633643,
|
|
"epoch": 0.3401360544217687,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.0006569559259332891,
|
|
"learning_rate": 6.621315192743764e-07,
|
|
"loss": 0.0384,
|
|
"num_tokens": 29355437.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.4253108948469162,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.41878482699394226,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 150,
|
|
"step_time": 609.9998593740165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3293.5,
|
|
"completions/mean_length": 1293.6328125,
|
|
"completions/mean_terminated_length": 729.4087524414062,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"entropy": 2.8541332334280014,
|
|
"epoch": 0.3424036281179138,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.271484375,
|
|
"kl": 0.0008383327276533237,
|
|
"learning_rate": 6.598639455782312e-07,
|
|
"loss": 0.0283,
|
|
"num_tokens": 29542890.0,
|
|
"reward": 0.12890625,
|
|
"reward_std": 0.47121794521808624,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.46915292739868164,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 151,
|
|
"step_time": 590.9694856717251
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3993.0,
|
|
"completions/mean_length": 1277.6484375,
|
|
"completions/mean_terminated_length": 932.8245239257812,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.154372226446867,
|
|
"epoch": 0.34467120181405897,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.0006259306728679803,
|
|
"learning_rate": 6.575963718820862e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 29729713.0,
|
|
"reward": 0.17578125,
|
|
"reward_std": 0.5465134978294373,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5087014436721802,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 152,
|
|
"step_time": 596.8687739442103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3735.5,
|
|
"completions/mean_length": 1387.640625,
|
|
"completions/mean_terminated_length": 969.6605834960938,
|
|
"completions/min_length": 6.5,
|
|
"completions/min_terminated_length": 6.5,
|
|
"entropy": 3.2348221354186535,
|
|
"epoch": 0.3469387755102041,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2314453125,
|
|
"kl": 0.0006377428417181363,
|
|
"learning_rate": 6.55328798185941e-07,
|
|
"loss": 0.0942,
|
|
"num_tokens": 29931883.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42124179005622864,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.41934312880039215,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 153,
|
|
"step_time": 596.5524867619388
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3774.5,
|
|
"completions/mean_length": 1128.9140625,
|
|
"completions/mean_terminated_length": 735.2179870605469,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 2.5647777430713177,
|
|
"epoch": 0.3492063492063492,
|
|
"frac_reward_zero_std": 0.96875,
|
|
"grad_norm": 0.12890625,
|
|
"kl": 0.0008062758497544564,
|
|
"learning_rate": 6.530612244897958e-07,
|
|
"loss": -0.0198,
|
|
"num_tokens": 30099304.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.19251982122659683,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 154,
|
|
"step_time": 596.5476731141098
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3758.0,
|
|
"completions/mean_length": 1193.40625,
|
|
"completions/mean_terminated_length": 893.1379089355469,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.0977296885102987,
|
|
"epoch": 0.35147392290249435,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.0006512100972031476,
|
|
"learning_rate": 6.507936507936507e-07,
|
|
"loss": -0.0559,
|
|
"num_tokens": 30276308.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4620004892349243,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 155,
|
|
"step_time": 614.8037619171664
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3900.5,
|
|
"completions/mean_length": 1234.3671875,
|
|
"completions/mean_terminated_length": 734.9114379882812,
|
|
"completions/min_length": 34.0,
|
|
"completions/min_terminated_length": 34.0,
|
|
"entropy": 3.0946367289870977,
|
|
"epoch": 0.35374149659863946,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1962890625,
|
|
"kl": 0.000800084431830328,
|
|
"learning_rate": 6.485260770975056e-07,
|
|
"loss": 0.015,
|
|
"num_tokens": 30460023.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.20951050519943237,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 156,
|
|
"step_time": 612.3484158357605
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3876.5,
|
|
"completions/mean_length": 1383.71875,
|
|
"completions/mean_terminated_length": 1050.631591796875,
|
|
"completions/min_length": 128.5,
|
|
"completions/min_terminated_length": 128.5,
|
|
"entropy": 3.786060569807887,
|
|
"epoch": 0.35600907029478457,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.000609957389315241,
|
|
"learning_rate": 6.462585034013605e-07,
|
|
"loss": 0.0422,
|
|
"num_tokens": 30661907.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5391381084918976,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.537759855389595,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 157,
|
|
"step_time": 604.4673374257982
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3754.5,
|
|
"completions/mean_length": 1192.28125,
|
|
"completions/mean_terminated_length": 924.2178039550781,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.8188612312078476,
|
|
"epoch": 0.35827664399092973,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.25390625,
|
|
"kl": 0.0006715908402838977,
|
|
"learning_rate": 6.439909297052154e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 30837539.0,
|
|
"reward": 0.10546875,
|
|
"reward_std": 0.41693858802318573,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 158,
|
|
"step_time": 607.1368471416645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3883.0,
|
|
"completions/mean_length": 1188.6171875,
|
|
"completions/mean_terminated_length": 834.0562133789062,
|
|
"completions/min_length": 41.5,
|
|
"completions/min_terminated_length": 41.5,
|
|
"entropy": 3.234034914523363,
|
|
"epoch": 0.36054421768707484,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2177734375,
|
|
"kl": 0.000756600022668863,
|
|
"learning_rate": 6.417233560090702e-07,
|
|
"loss": 0.028,
|
|
"num_tokens": 31012198.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.22127514332532883,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 159,
|
|
"step_time": 597.2393137025647
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3752.0,
|
|
"completions/mean_length": 1580.6953125,
|
|
"completions/mean_terminated_length": 1029.663330078125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.8659055149182677,
|
|
"epoch": 0.36281179138321995,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1552734375,
|
|
"kl": 0.0006428698948184319,
|
|
"learning_rate": 6.394557823129252e-07,
|
|
"loss": 0.027,
|
|
"num_tokens": 31237771.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.25989100337028503,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 160,
|
|
"step_time": 593.4085997003131
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3540.0,
|
|
"completions/mean_length": 1250.578125,
|
|
"completions/mean_terminated_length": 785.9830932617188,
|
|
"completions/min_length": 9.5,
|
|
"completions/min_terminated_length": 9.5,
|
|
"entropy": 2.568399177864194,
|
|
"epoch": 0.36507936507936506,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2255859375,
|
|
"kl": 0.000799570717390452,
|
|
"learning_rate": 6.3718820861678e-07,
|
|
"loss": 0.0648,
|
|
"num_tokens": 31421905.0,
|
|
"reward": 0.13671875,
|
|
"reward_std": 0.48334628343582153,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.46915292739868164,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 161,
|
|
"step_time": 609.9772143028677
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3961.5,
|
|
"completions/mean_length": 1285.578125,
|
|
"completions/mean_terminated_length": 970.8536987304688,
|
|
"completions/min_length": 62.5,
|
|
"completions/min_terminated_length": 62.5,
|
|
"entropy": 3.558231770992279,
|
|
"epoch": 0.3673469387755102,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2451171875,
|
|
"kl": 0.0007242586889333325,
|
|
"learning_rate": 6.349206349206349e-07,
|
|
"loss": 0.0122,
|
|
"num_tokens": 31613619.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 162,
|
|
"step_time": 618.7352644796483
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3681.0,
|
|
"completions/mean_length": 1219.1875,
|
|
"completions/mean_terminated_length": 806.0791015625,
|
|
"completions/min_length": 60.5,
|
|
"completions/min_terminated_length": 60.5,
|
|
"entropy": 3.231019349768758,
|
|
"epoch": 0.36961451247165533,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.0007710261102147342,
|
|
"learning_rate": 6.326530612244898e-07,
|
|
"loss": 0.0115,
|
|
"num_tokens": 31794251.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.4038149565458298,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 163,
|
|
"step_time": 611.0094007700682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4046.0,
|
|
"completions/mean_length": 1147.7109375,
|
|
"completions/mean_terminated_length": 902.7987976074219,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 3.6982441302388906,
|
|
"epoch": 0.37188208616780044,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.310546875,
|
|
"kl": 0.0006015987819409929,
|
|
"learning_rate": 6.303854875283446e-07,
|
|
"loss": 0.0619,
|
|
"num_tokens": 31965406.0,
|
|
"reward": 0.19140625,
|
|
"reward_std": 0.5857243835926056,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.585066169500351,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 164,
|
|
"step_time": 604.1481093526818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3206.0,
|
|
"completions/mean_length": 1135.171875,
|
|
"completions/mean_terminated_length": 889.2405395507812,
|
|
"completions/min_length": 46.5,
|
|
"completions/min_terminated_length": 46.5,
|
|
"entropy": 3.518560167402029,
|
|
"epoch": 0.3741496598639456,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33203125,
|
|
"kl": 0.0007305615276891331,
|
|
"learning_rate": 6.281179138321994e-07,
|
|
"loss": 0.0708,
|
|
"num_tokens": 32134172.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4875367283821106,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48353175818920135,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 165,
|
|
"step_time": 608.5186735871248
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3831.0,
|
|
"completions/mean_length": 1207.3203125,
|
|
"completions/mean_terminated_length": 821.7534790039062,
|
|
"completions/min_length": 78.5,
|
|
"completions/min_terminated_length": 78.5,
|
|
"entropy": 2.9473390579223633,
|
|
"epoch": 0.3764172335600907,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2001953125,
|
|
"kl": 0.0008242658777817269,
|
|
"learning_rate": 6.258503401360544e-07,
|
|
"loss": 0.0249,
|
|
"num_tokens": 32312321.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35475611686706543,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 166,
|
|
"step_time": 591.5427933516912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3560.5,
|
|
"completions/mean_length": 1332.3828125,
|
|
"completions/mean_terminated_length": 820.6018676757812,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.1136458199471235,
|
|
"epoch": 0.3786848072562358,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2255859375,
|
|
"kl": 0.0008250123260040709,
|
|
"learning_rate": 6.235827664399092e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 32508226.0,
|
|
"reward": 0.17578125,
|
|
"reward_std": 0.5704257786273956,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.537759855389595,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 167,
|
|
"step_time": 614.4076505717821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3871.5,
|
|
"completions/mean_length": 1342.03125,
|
|
"completions/mean_terminated_length": 951.0034790039062,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.266963863745332,
|
|
"epoch": 0.38095238095238093,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2412109375,
|
|
"kl": 0.0006693732443636691,
|
|
"learning_rate": 6.213151927437641e-07,
|
|
"loss": 0.0323,
|
|
"num_tokens": 32704418.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.45891571044921875,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 168,
|
|
"step_time": 624.9228283683769
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3467.0,
|
|
"completions/mean_length": 1220.078125,
|
|
"completions/mean_terminated_length": 779.7297058105469,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.179447118192911,
|
|
"epoch": 0.3832199546485261,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2197265625,
|
|
"kl": 0.0008172887933142192,
|
|
"learning_rate": 6.19047619047619e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 32883868.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.34974396228790283,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 169,
|
|
"step_time": 597.6843538912944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3949.5,
|
|
"completions/mean_length": 1473.78125,
|
|
"completions/mean_terminated_length": 893.5779418945312,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 4.046913323923945,
|
|
"epoch": 0.3854875283446712,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.19140625,
|
|
"kl": 0.0007520312719861977,
|
|
"learning_rate": 6.167800453514739e-07,
|
|
"loss": 0.0336,
|
|
"num_tokens": 33099308.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31276239454746246,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 170,
|
|
"step_time": 641.5558594134636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3863.5,
|
|
"completions/mean_length": 1152.2265625,
|
|
"completions/mean_terminated_length": 791.8938293457031,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 2.786166712641716,
|
|
"epoch": 0.3877551020408163,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.0008361312511624419,
|
|
"learning_rate": 6.145124716553288e-07,
|
|
"loss": 0.0408,
|
|
"num_tokens": 33270145.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.40065382421016693,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 171,
|
|
"step_time": 589.4972572457045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3692.0,
|
|
"completions/mean_length": 1298.7578125,
|
|
"completions/mean_terminated_length": 747.3540344238281,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 2.450922917574644,
|
|
"epoch": 0.3900226757369615,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.154296875,
|
|
"kl": 0.0007833251165720867,
|
|
"learning_rate": 6.122448979591837e-07,
|
|
"loss": 0.013,
|
|
"num_tokens": 33460146.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.23264852166175842,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 172,
|
|
"step_time": 603.4941355958581
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3876.5,
|
|
"completions/mean_length": 1535.6171875,
|
|
"completions/mean_terminated_length": 913.689697265625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.8065616469830275,
|
|
"epoch": 0.3922902494331066,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.0007675115180063585,
|
|
"learning_rate": 6.099773242630385e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 33679733.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4497220069169998,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.44585782289505005,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 173,
|
|
"step_time": 590.0731657166034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3337.5,
|
|
"completions/mean_length": 1085.921875,
|
|
"completions/mean_terminated_length": 745.787353515625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.998498792760074,
|
|
"epoch": 0.3945578231292517,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.0008980963193607749,
|
|
"learning_rate": 6.077097505668934e-07,
|
|
"loss": -0.0153,
|
|
"num_tokens": 33843183.0,
|
|
"reward": 0.12109375,
|
|
"reward_std": 0.42828261852264404,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.41878482699394226,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 174,
|
|
"step_time": 609.4836022700183
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4021.0,
|
|
"completions/mean_length": 1373.9375,
|
|
"completions/mean_terminated_length": 927.4676208496094,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.933150317519903,
|
|
"epoch": 0.3968253968253968,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1962890625,
|
|
"kl": 0.0007073812794260448,
|
|
"learning_rate": 6.054421768707482e-07,
|
|
"loss": 0.0275,
|
|
"num_tokens": 34043103.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.4465160369873047,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.44585782289505005,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 175,
|
|
"step_time": 591.875461160671
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3870.0,
|
|
"completions/mean_length": 1442.90625,
|
|
"completions/mean_terminated_length": 864.3609924316406,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.1040230710059404,
|
|
"epoch": 0.39909297052154197,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.240234375,
|
|
"kl": 0.0009050897319866635,
|
|
"learning_rate": 6.031746031746031e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 34253087.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 176,
|
|
"step_time": 600.0417243833654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3609.5,
|
|
"completions/mean_length": 1288.234375,
|
|
"completions/mean_terminated_length": 860.0355834960938,
|
|
"completions/min_length": 39.0,
|
|
"completions/min_terminated_length": 39.0,
|
|
"entropy": 3.137294977903366,
|
|
"epoch": 0.4013605442176871,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.0007529877266279073,
|
|
"learning_rate": 6.00907029478458e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 34441425.0,
|
|
"reward": 0.02734375,
|
|
"reward_std": 0.17220696061849594,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 177,
|
|
"step_time": 593.9724750495516
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3827.5,
|
|
"completions/mean_length": 1543.2421875,
|
|
"completions/mean_terminated_length": 1122.2916564941406,
|
|
"completions/min_length": 33.5,
|
|
"completions/min_terminated_length": 33.5,
|
|
"entropy": 3.8696006890386343,
|
|
"epoch": 0.4036281179138322,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.16796875,
|
|
"kl": 0.0006700336380163208,
|
|
"learning_rate": 5.986394557823129e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 34662320.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2628627270460129,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 178,
|
|
"step_time": 591.8633469510823
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3894.5,
|
|
"completions/mean_length": 1429.3515625,
|
|
"completions/mean_terminated_length": 913.3193054199219,
|
|
"completions/min_length": 17.5,
|
|
"completions/min_terminated_length": 17.5,
|
|
"entropy": 3.6077201571315527,
|
|
"epoch": 0.40589569160997735,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.0007685890332140843,
|
|
"learning_rate": 5.963718820861677e-07,
|
|
"loss": -0.033,
|
|
"num_tokens": 34868437.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.2787574231624603,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 179,
|
|
"step_time": 604.5654797428288
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3793.5,
|
|
"completions/mean_length": 1134.96875,
|
|
"completions/mean_terminated_length": 937.5667114257812,
|
|
"completions/min_length": 89.5,
|
|
"completions/min_terminated_length": 89.5,
|
|
"entropy": 4.224542632699013,
|
|
"epoch": 0.40816326530612246,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30078125,
|
|
"kl": 0.0007521463867306011,
|
|
"learning_rate": 5.941043083900227e-07,
|
|
"loss": 0.07,
|
|
"num_tokens": 35039037.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.45891571044921875,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 180,
|
|
"step_time": 599.8329591401853
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3483.5,
|
|
"completions/mean_length": 1394.2265625,
|
|
"completions/mean_terminated_length": 865.4013977050781,
|
|
"completions/min_length": 44.5,
|
|
"completions/min_terminated_length": 44.5,
|
|
"entropy": 3.0462828669697046,
|
|
"epoch": 0.41043083900226757,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.177734375,
|
|
"kl": 0.0006654558101217845,
|
|
"learning_rate": 5.918367346938775e-07,
|
|
"loss": -0.0061,
|
|
"num_tokens": 35244918.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.39760591089725494,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 181,
|
|
"step_time": 646.8893661289476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3723.5,
|
|
"completions/mean_length": 1218.859375,
|
|
"completions/mean_terminated_length": 774.5813293457031,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 3.358133727684617,
|
|
"epoch": 0.4126984126984127,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.248046875,
|
|
"kl": 0.000827424693852663,
|
|
"learning_rate": 5.895691609977324e-07,
|
|
"loss": 0.0624,
|
|
"num_tokens": 35425864.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.3458264470100403,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.3145764470100403,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 182,
|
|
"step_time": 603.8318180949427
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0546875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3917.5,
|
|
"completions/mean_length": 1104.1875,
|
|
"completions/mean_terminated_length": 931.0534057617188,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.261079495772719,
|
|
"epoch": 0.41496598639455784,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1787109375,
|
|
"kl": 0.000713553286914248,
|
|
"learning_rate": 5.873015873015873e-07,
|
|
"loss": 0.0111,
|
|
"num_tokens": 35592608.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3911572992801666,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 183,
|
|
"step_time": 610.3628411479294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3972.0,
|
|
"completions/mean_length": 1588.46875,
|
|
"completions/mean_terminated_length": 1039.0706176757812,
|
|
"completions/min_length": 45.5,
|
|
"completions/min_terminated_length": 45.5,
|
|
"entropy": 3.6161519903689623,
|
|
"epoch": 0.41723356009070295,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2333984375,
|
|
"kl": 0.000688317676576844,
|
|
"learning_rate": 5.850340136054421e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 35819992.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.3480724096298218,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.29378482699394226,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 184,
|
|
"step_time": 598.0165545996279
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3774.0,
|
|
"completions/mean_length": 1257.4765625,
|
|
"completions/mean_terminated_length": 794.9091796875,
|
|
"completions/min_length": 19.0,
|
|
"completions/min_terminated_length": 19.0,
|
|
"entropy": 3.368335533887148,
|
|
"epoch": 0.41950113378684806,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.14453125,
|
|
"kl": 0.0008394297783524962,
|
|
"learning_rate": 5.82766439909297e-07,
|
|
"loss": 0.0063,
|
|
"num_tokens": 36006889.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.19491341710090637,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 185,
|
|
"step_time": 621.8880987530574
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3889.5,
|
|
"completions/mean_length": 1315.9609375,
|
|
"completions/mean_terminated_length": 1023.0173034667969,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 4.1564923617988825,
|
|
"epoch": 0.4217687074829932,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.201171875,
|
|
"kl": 0.0006244404175959062,
|
|
"learning_rate": 5.804988662131519e-07,
|
|
"loss": -0.0171,
|
|
"num_tokens": 36198388.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.3068941682577133,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 186,
|
|
"step_time": 598.4586103376932
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3769.5,
|
|
"completions/mean_length": 1304.609375,
|
|
"completions/mean_terminated_length": 907.1460876464844,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.8947392273694277,
|
|
"epoch": 0.42403628117913833,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.0006855657302367035,
|
|
"learning_rate": 5.782312925170067e-07,
|
|
"loss": -0.0455,
|
|
"num_tokens": 36390730.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4261941611766815,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.41934312880039215,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 187,
|
|
"step_time": 604.2838142933324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3703.5,
|
|
"completions/mean_length": 1432.546875,
|
|
"completions/mean_terminated_length": 890.0086059570312,
|
|
"completions/min_length": 79.5,
|
|
"completions/min_terminated_length": 79.5,
|
|
"entropy": 3.796390214934945,
|
|
"epoch": 0.42630385487528344,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.19921875,
|
|
"kl": 0.0008078004575509112,
|
|
"learning_rate": 5.759637188208617e-07,
|
|
"loss": -0.0111,
|
|
"num_tokens": 36600744.0,
|
|
"reward": 0.14453125,
|
|
"reward_std": 0.5163404941558838,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.46915292739868164,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 188,
|
|
"step_time": 644.0010008728132
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3854.5,
|
|
"completions/mean_length": 1044.3046875,
|
|
"completions/mean_terminated_length": 710.635986328125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.0846201702952385,
|
|
"epoch": 0.42857142857142855,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.00092321831107256,
|
|
"learning_rate": 5.736961451247165e-07,
|
|
"loss": 0.0387,
|
|
"num_tokens": 36756555.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.23264852166175842,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 189,
|
|
"step_time": 590.7555743027478
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3966.0,
|
|
"completions/mean_length": 1318.2421875,
|
|
"completions/mean_terminated_length": 980.7483825683594,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.991009619086981,
|
|
"epoch": 0.4308390022675737,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.25390625,
|
|
"kl": 0.0007133439785320661,
|
|
"learning_rate": 5.714285714285714e-07,
|
|
"loss": 0.0719,
|
|
"num_tokens": 36948194.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.44924473762512207,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 190,
|
|
"step_time": 594.243725836277
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3874.0,
|
|
"completions/mean_length": 1531.9765625,
|
|
"completions/mean_terminated_length": 939.894775390625,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"entropy": 3.7708285078406334,
|
|
"epoch": 0.4331065759637188,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1572265625,
|
|
"kl": 0.0007431543922393757,
|
|
"learning_rate": 5.691609977324263e-07,
|
|
"loss": 0.0578,
|
|
"num_tokens": 37169635.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.17536809295415878,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 191,
|
|
"step_time": 615.3594089960679
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3857.0,
|
|
"completions/mean_length": 1387.3203125,
|
|
"completions/mean_terminated_length": 974.0755615234375,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 4.500912141054869,
|
|
"epoch": 0.43537414965986393,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2470703125,
|
|
"kl": 0.0007893707388575422,
|
|
"learning_rate": 5.668934240362812e-07,
|
|
"loss": 0.0566,
|
|
"num_tokens": 37371768.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.27712368965148926,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 192,
|
|
"step_time": 595.7811410389841
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4008.0,
|
|
"completions/mean_length": 1379.625,
|
|
"completions/mean_terminated_length": 1046.0350952148438,
|
|
"completions/min_length": 10.5,
|
|
"completions/min_terminated_length": 10.5,
|
|
"entropy": 4.2326104789972305,
|
|
"epoch": 0.4376417233560091,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.000629605076937878,
|
|
"learning_rate": 5.64625850340136e-07,
|
|
"loss": -0.0227,
|
|
"num_tokens": 37569528.0,
|
|
"reward": 0.12890625,
|
|
"reward_std": 0.4429413825273514,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.4395764470100403,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 193,
|
|
"step_time": 589.9761042450555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4029.0,
|
|
"completions/mean_length": 1510.1953125,
|
|
"completions/mean_terminated_length": 1008.45556640625,
|
|
"completions/min_length": 7.5,
|
|
"completions/min_terminated_length": 7.5,
|
|
"entropy": 3.4559367783367634,
|
|
"epoch": 0.4399092970521542,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2392578125,
|
|
"kl": 0.0008663589715069975,
|
|
"learning_rate": 5.62358276643991e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 37789069.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36236928403377533,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 194,
|
|
"step_time": 619.8996191667393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3382.5,
|
|
"completions/mean_length": 1188.1328125,
|
|
"completions/mean_terminated_length": 885.4705810546875,
|
|
"completions/min_length": 63.0,
|
|
"completions/min_terminated_length": 63.0,
|
|
"entropy": 3.577680381014943,
|
|
"epoch": 0.4421768707482993,
|
|
"frac_reward_zero_std": 0.71875,
|
|
"grad_norm": 0.28125,
|
|
"kl": 0.0007030921101431886,
|
|
"learning_rate": 5.600907029478457e-07,
|
|
"loss": 0.009,
|
|
"num_tokens": 37963586.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4616628885269165,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 195,
|
|
"step_time": 597.3808077289723
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3121.5,
|
|
"completions/mean_length": 1289.8515625,
|
|
"completions/mean_terminated_length": 857.8040466308594,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.255021285265684,
|
|
"epoch": 0.4444444444444444,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.251953125,
|
|
"kl": 0.0008791069922153838,
|
|
"learning_rate": 5.578231292517006e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 38150623.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.35028699040412903,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 196,
|
|
"step_time": 594.9530698563904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3980.0,
|
|
"completions/mean_length": 1286.8046875,
|
|
"completions/mean_terminated_length": 856.8700256347656,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.6953976415097713,
|
|
"epoch": 0.4467120181405896,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.158203125,
|
|
"kl": 0.0008147612952598138,
|
|
"learning_rate": 5.555555555555555e-07,
|
|
"loss": -0.0055,
|
|
"num_tokens": 38339362.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.35834936797618866,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.06099375709891319,
|
|
"step": 197,
|
|
"step_time": 619.3629526048899
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3937.5,
|
|
"completions/mean_length": 1289.8125,
|
|
"completions/mean_terminated_length": 890.4508361816406,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.45363543368876,
|
|
"epoch": 0.4489795918367347,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.0007131813167688961,
|
|
"learning_rate": 5.532879818594104e-07,
|
|
"loss": 0.0297,
|
|
"num_tokens": 38528238.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.35433506965637207,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 198,
|
|
"step_time": 621.3943899697624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3976.0,
|
|
"completions/mean_length": 1519.2578125,
|
|
"completions/mean_terminated_length": 955.14697265625,
|
|
"completions/min_length": 92.5,
|
|
"completions/min_terminated_length": 92.5,
|
|
"entropy": 3.6701013687998056,
|
|
"epoch": 0.4512471655328798,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.07373046875,
|
|
"kl": 0.0007340602478507208,
|
|
"learning_rate": 5.510204081632652e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 38752167.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.32732683420181274,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 199,
|
|
"step_time": 653.9326698929071
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3888.0,
|
|
"completions/mean_length": 1356.78125,
|
|
"completions/mean_terminated_length": 845.1145629882812,
|
|
"completions/min_length": 80.5,
|
|
"completions/min_terminated_length": 80.5,
|
|
"entropy": 3.3553403858095407,
|
|
"epoch": 0.45351473922902497,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.236328125,
|
|
"kl": 0.0009613324955353164,
|
|
"learning_rate": 5.487528344671202e-07,
|
|
"loss": 0.0274,
|
|
"num_tokens": 38948379.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5335856229066849,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5087014436721802,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 200,
|
|
"step_time": 599.1757256691344
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3569.5,
|
|
"completions/mean_length": 1156.234375,
|
|
"completions/mean_terminated_length": 736.2678833007812,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 3.024919604882598,
|
|
"epoch": 0.4557823129251701,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.0008046829734666971,
|
|
"learning_rate": 5.46485260770975e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 39120353.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3085011765360832,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 201,
|
|
"step_time": 601.9984689583071
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4037.0,
|
|
"completions/mean_length": 1225.328125,
|
|
"completions/mean_terminated_length": 786.4696350097656,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.410710232332349,
|
|
"epoch": 0.4580498866213152,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.0008340677950400277,
|
|
"learning_rate": 5.4421768707483e-07,
|
|
"loss": 0.0144,
|
|
"num_tokens": 39302347.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.33031322062015533,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 202,
|
|
"step_time": 604.1163647472858
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3566.5,
|
|
"completions/mean_length": 1245.15625,
|
|
"completions/mean_terminated_length": 946.5691223144531,
|
|
"completions/min_length": 95.5,
|
|
"completions/min_terminated_length": 95.5,
|
|
"entropy": 3.3697747830301523,
|
|
"epoch": 0.4603174603174603,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.203125,
|
|
"kl": 0.0006801806080147799,
|
|
"learning_rate": 5.419501133786848e-07,
|
|
"loss": 0.022,
|
|
"num_tokens": 39486435.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5095740258693695,
|
|
"rewards/accuracy_reward_func/mean": 0.140625,
|
|
"rewards/accuracy_reward_func/std": 0.506826862692833,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 203,
|
|
"step_time": 607.9280528882518
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3747.0,
|
|
"completions/mean_length": 1095.71875,
|
|
"completions/mean_terminated_length": 811.7057189941406,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.2434293814003468,
|
|
"epoch": 0.46258503401360546,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.24609375,
|
|
"kl": 0.0008007425558389514,
|
|
"learning_rate": 5.396825396825396e-07,
|
|
"loss": 0.0084,
|
|
"num_tokens": 39651447.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43452155590057373,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 204,
|
|
"step_time": 621.2008724869229
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3907.0,
|
|
"completions/mean_length": 1477.6953125,
|
|
"completions/mean_terminated_length": 995.0191040039062,
|
|
"completions/min_length": 71.5,
|
|
"completions/min_terminated_length": 71.5,
|
|
"entropy": 3.382938737049699,
|
|
"epoch": 0.46485260770975056,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.150390625,
|
|
"kl": 0.0008737306570765213,
|
|
"learning_rate": 5.374149659863945e-07,
|
|
"loss": 0.0192,
|
|
"num_tokens": 39864272.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.21063801646232605,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 205,
|
|
"step_time": 601.140031487681
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3876.5,
|
|
"completions/mean_length": 1396.5859375,
|
|
"completions/mean_terminated_length": 900.1669311523438,
|
|
"completions/min_length": 43.5,
|
|
"completions/min_terminated_length": 43.5,
|
|
"entropy": 3.881635056808591,
|
|
"epoch": 0.4671201814058957,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.11572265625,
|
|
"kl": 0.0007193161281975335,
|
|
"learning_rate": 5.351473922902494e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 40064735.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 206,
|
|
"step_time": 588.42849458009
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3696.0,
|
|
"completions/mean_length": 1242.1171875,
|
|
"completions/mean_terminated_length": 887.659912109375,
|
|
"completions/min_length": 118.5,
|
|
"completions/min_terminated_length": 118.5,
|
|
"entropy": 3.77775052562356,
|
|
"epoch": 0.46938775510204084,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.30078125,
|
|
"kl": 0.0006713231250614626,
|
|
"learning_rate": 5.328798185941042e-07,
|
|
"loss": 0.0163,
|
|
"num_tokens": 40247230.0,
|
|
"reward": 0.16015625,
|
|
"reward_std": 0.5177302062511444,
|
|
"rewards/accuracy_reward_func/mean": 0.140625,
|
|
"rewards/accuracy_reward_func/std": 0.5144647508859634,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 207,
|
|
"step_time": 594.75569937611
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4004.0,
|
|
"completions/mean_length": 1157.359375,
|
|
"completions/mean_terminated_length": 791.9639282226562,
|
|
"completions/min_length": 40.5,
|
|
"completions/min_terminated_length": 40.5,
|
|
"entropy": 3.5107192117720842,
|
|
"epoch": 0.47165532879818595,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2392578125,
|
|
"kl": 0.0007464863083441742,
|
|
"learning_rate": 5.306122448979592e-07,
|
|
"loss": 0.0053,
|
|
"num_tokens": 40420464.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5250208377838135,
|
|
"rewards/accuracy_reward_func/mean": 0.140625,
|
|
"rewards/accuracy_reward_func/std": 0.506826862692833,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 208,
|
|
"step_time": 619.3611796610057
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3810.5,
|
|
"completions/mean_length": 1336.046875,
|
|
"completions/mean_terminated_length": 914.0532531738281,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 3.636397385969758,
|
|
"epoch": 0.47392290249433106,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1533203125,
|
|
"kl": 0.0007521043035012553,
|
|
"learning_rate": 5.28344671201814e-07,
|
|
"loss": 0.0227,
|
|
"num_tokens": 40613726.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3689750134944916,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 209,
|
|
"step_time": 597.7159785535187
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3910.0,
|
|
"completions/mean_length": 1269.078125,
|
|
"completions/mean_terminated_length": 919.72314453125,
|
|
"completions/min_length": 75.0,
|
|
"completions/min_terminated_length": 75.0,
|
|
"entropy": 4.198507905006409,
|
|
"epoch": 0.47619047619047616,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.0007738635922578396,
|
|
"learning_rate": 5.260770975056689e-07,
|
|
"loss": 0.0628,
|
|
"num_tokens": 40800436.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3918892741203308,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 210,
|
|
"step_time": 616.5440966836177
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4029.5,
|
|
"completions/mean_length": 1283.0078125,
|
|
"completions/mean_terminated_length": 936.6346740722656,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.8082601577043533,
|
|
"epoch": 0.47845804988662133,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2451171875,
|
|
"kl": 0.0008697739212948363,
|
|
"learning_rate": 5.238095238095238e-07,
|
|
"loss": -0.0045,
|
|
"num_tokens": 40987337.0,
|
|
"reward": 0.10546875,
|
|
"reward_std": 0.42209140956401825,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.41934312880039215,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 211,
|
|
"step_time": 601.1385076148435
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3623.5,
|
|
"completions/mean_length": 1430.578125,
|
|
"completions/mean_terminated_length": 1047.4884948730469,
|
|
"completions/min_length": 76.0,
|
|
"completions/min_terminated_length": 76.0,
|
|
"entropy": 3.9115954656153917,
|
|
"epoch": 0.48072562358276644,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1611328125,
|
|
"kl": 0.0007129267823984264,
|
|
"learning_rate": 5.215419501133787e-07,
|
|
"loss": 0.0052,
|
|
"num_tokens": 41193523.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.3773163855075836,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 212,
|
|
"step_time": 590.1808690954931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3862.5,
|
|
"completions/mean_length": 1164.9375,
|
|
"completions/mean_terminated_length": 749.9704284667969,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 3.7405852507799864,
|
|
"epoch": 0.48299319727891155,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.0009099668341150391,
|
|
"learning_rate": 5.192743764172335e-07,
|
|
"loss": 0.0343,
|
|
"num_tokens": 41366227.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4620004892349243,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 213,
|
|
"step_time": 604.5149142243899
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3565.0,
|
|
"completions/mean_length": 1391.234375,
|
|
"completions/mean_terminated_length": 979.6613464355469,
|
|
"completions/min_length": 15.5,
|
|
"completions/min_terminated_length": 15.5,
|
|
"entropy": 3.8633985333144665,
|
|
"epoch": 0.4852607709750567,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.11767578125,
|
|
"kl": 0.0007512560832765303,
|
|
"learning_rate": 5.170068027210885e-07,
|
|
"loss": 0.0235,
|
|
"num_tokens": 41568249.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.3003681004047394,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 214,
|
|
"step_time": 602.045600481797
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4081.0,
|
|
"completions/mean_length": 1222.78125,
|
|
"completions/mean_terminated_length": 929.8265075683594,
|
|
"completions/min_length": 14.5,
|
|
"completions/min_terminated_length": 14.5,
|
|
"entropy": 3.546670449897647,
|
|
"epoch": 0.4875283446712018,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1943359375,
|
|
"kl": 0.0008797261202744266,
|
|
"learning_rate": 5.147392290249432e-07,
|
|
"loss": 0.0137,
|
|
"num_tokens": 41748257.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34415413439273834,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 215,
|
|
"step_time": 598.8036100231111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3936.0,
|
|
"completions/mean_length": 1250.7265625,
|
|
"completions/mean_terminated_length": 1012.4807739257812,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"entropy": 4.179215501062572,
|
|
"epoch": 0.4897959183673469,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.0007071993468343862,
|
|
"learning_rate": 5.124716553287982e-07,
|
|
"loss": -0.0274,
|
|
"num_tokens": 41932066.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3918892741203308,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 216,
|
|
"step_time": 596.9903638330288
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3648.5,
|
|
"completions/mean_length": 1094.8359375,
|
|
"completions/mean_terminated_length": 781.5816040039062,
|
|
"completions/min_length": 18.5,
|
|
"completions/min_terminated_length": 18.5,
|
|
"entropy": 3.0233175065368414,
|
|
"epoch": 0.49206349206349204,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.177734375,
|
|
"kl": 0.0009479731256760715,
|
|
"learning_rate": 5.10204081632653e-07,
|
|
"loss": 0.0437,
|
|
"num_tokens": 42098549.0,
|
|
"reward": 0.01171875,
|
|
"reward_std": 0.07509202510118484,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 217,
|
|
"step_time": 615.2984105711803
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3910.5,
|
|
"completions/mean_length": 1372.5078125,
|
|
"completions/mean_terminated_length": 1037.5884094238281,
|
|
"completions/min_length": 19.5,
|
|
"completions/min_terminated_length": 19.5,
|
|
"entropy": 4.472298253327608,
|
|
"epoch": 0.4943310657596372,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1611328125,
|
|
"kl": 0.0007240536115205032,
|
|
"learning_rate": 5.079365079365079e-07,
|
|
"loss": 0.0222,
|
|
"num_tokens": 42297406.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.21445615589618683,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 218,
|
|
"step_time": 597.1894538952038
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3758.5,
|
|
"completions/mean_length": 1535.8828125,
|
|
"completions/mean_terminated_length": 819.2635192871094,
|
|
"completions/min_length": 60.5,
|
|
"completions/min_terminated_length": 60.5,
|
|
"entropy": 3.3840507585555315,
|
|
"epoch": 0.4965986394557823,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1318359375,
|
|
"kl": 0.0010230302450509043,
|
|
"learning_rate": 5.056689342403628e-07,
|
|
"loss": 0.0098,
|
|
"num_tokens": 42522991.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.16574779152870178,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 219,
|
|
"step_time": 661.3171017216519
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3945.5,
|
|
"completions/mean_length": 1314.3671875,
|
|
"completions/mean_terminated_length": 1027.8166809082031,
|
|
"completions/min_length": 7.5,
|
|
"completions/min_terminated_length": 7.5,
|
|
"entropy": 4.2411610670387745,
|
|
"epoch": 0.4988662131519274,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.30078125,
|
|
"kl": 0.0007040352243166126,
|
|
"learning_rate": 5.034013605442177e-07,
|
|
"loss": 0.0005,
|
|
"num_tokens": 42715702.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.46772849559783936,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.44585782289505005,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 220,
|
|
"step_time": 606.7818779926747
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 441,
|
|
"num_input_tokens_seen": 42715702,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|