7dc4c29a8e
Model: azalahmadkhan/Llama-3.2-3B-Instruct-GRPO-vanilla-G-4-novllm-25pct Source: Original Platform
3335 lines
123 KiB
JSON
3335 lines
123 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.2494331065759637,
|
|
"eval_steps": 500,
|
|
"global_step": 110,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3681.0,
|
|
"completions/mean_length": 1268.2265625,
|
|
"completions/mean_terminated_length": 980.7562866210938,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"entropy": 4.420659868046641,
|
|
"epoch": 0.0022675736961451248,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.19921875,
|
|
"kl": 0.0003264336364736664,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0093,
|
|
"num_tokens": 184821.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31472553312778473,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 1,
|
|
"step_time": 600.7805640818551
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3984.0,
|
|
"completions/mean_length": 1622.3203125,
|
|
"completions/mean_terminated_length": 962.6857299804688,
|
|
"completions/min_length": 107.5,
|
|
"completions/min_terminated_length": 107.5,
|
|
"entropy": 3.2865531742572784,
|
|
"epoch": 0.0045351473922902496,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.09375,
|
|
"kl": 0.0003618000596361526,
|
|
"learning_rate": 9.977324263038547e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 417594.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2519752085208893,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 2,
|
|
"step_time": 603.3548571760766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3468.5,
|
|
"completions/mean_length": 1188.703125,
|
|
"completions/mean_terminated_length": 803.1010437011719,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 2.9000580459833145,
|
|
"epoch": 0.006802721088435374,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.228515625,
|
|
"kl": 0.0003971439623455808,
|
|
"learning_rate": 9.954648526077097e-07,
|
|
"loss": -0.0146,
|
|
"num_tokens": 592512.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.2943021357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 3,
|
|
"step_time": 592.7558827972971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3806.5,
|
|
"completions/mean_length": 1708.2734375,
|
|
"completions/mean_terminated_length": 1070.256103515625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.978165291249752,
|
|
"epoch": 0.009070294784580499,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.134765625,
|
|
"kl": 0.0003516419765219325,
|
|
"learning_rate": 9.931972789115645e-07,
|
|
"loss": 0.0028,
|
|
"num_tokens": 835279.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 4,
|
|
"step_time": 592.1812793556601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3869.0,
|
|
"completions/mean_length": 1158.1484375,
|
|
"completions/mean_terminated_length": 680.8644409179688,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 2.368096474558115,
|
|
"epoch": 0.011337868480725623,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.00045219186085887486,
|
|
"learning_rate": 9.909297052154193e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 1006302.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.2582981809973717,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 5,
|
|
"step_time": 594.9633022211492
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3758.0,
|
|
"completions/mean_length": 1112.7421875,
|
|
"completions/mean_terminated_length": 805.9321899414062,
|
|
"completions/min_length": 1.5,
|
|
"completions/min_terminated_length": 1.5,
|
|
"entropy": 3.4849978666752577,
|
|
"epoch": 0.013605442176870748,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.26953125,
|
|
"kl": 0.0004078899446540163,
|
|
"learning_rate": 9.886621315192743e-07,
|
|
"loss": 0.026,
|
|
"num_tokens": 1177829.0,
|
|
"reward": 0.14453125,
|
|
"reward_std": 0.5158129632472992,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48353175818920135,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 6,
|
|
"step_time": 635.190734772943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3845.5,
|
|
"completions/mean_length": 1290.9140625,
|
|
"completions/mean_terminated_length": 861.3218994140625,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.505165945738554,
|
|
"epoch": 0.015873015873015872,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.00037227333541522967,
|
|
"learning_rate": 9.86394557823129e-07,
|
|
"loss": 0.035,
|
|
"num_tokens": 1365610.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3109140843153,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 7,
|
|
"step_time": 591.5275562573224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4059.5,
|
|
"completions/mean_length": 1521.6640625,
|
|
"completions/mean_terminated_length": 1102.5285034179688,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.9575814120471478,
|
|
"epoch": 0.018140589569160998,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1513671875,
|
|
"kl": 0.0003584225587474066,
|
|
"learning_rate": 9.84126984126984e-07,
|
|
"loss": 0.0205,
|
|
"num_tokens": 1586235.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.20868311822414398,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 8,
|
|
"step_time": 610.3726020189933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3902.5,
|
|
"completions/mean_length": 1471.5859375,
|
|
"completions/mean_terminated_length": 957.7996215820312,
|
|
"completions/min_length": 25.5,
|
|
"completions/min_terminated_length": 25.5,
|
|
"entropy": 3.6384904142469168,
|
|
"epoch": 0.02040816326530612,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2236328125,
|
|
"kl": 0.00037756811252620537,
|
|
"learning_rate": 9.818594104308389e-07,
|
|
"loss": 0.0225,
|
|
"num_tokens": 1797770.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3733227699995041,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 9,
|
|
"step_time": 596.3971076705493
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3439.0,
|
|
"completions/mean_length": 1379.828125,
|
|
"completions/mean_terminated_length": 848.1103515625,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 4.184969509020448,
|
|
"epoch": 0.022675736961451247,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.00037742951417385484,
|
|
"learning_rate": 9.795918367346939e-07,
|
|
"loss": 0.0448,
|
|
"num_tokens": 2000228.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.40065382421016693,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 10,
|
|
"step_time": 614.9045870057307
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4059.5,
|
|
"completions/mean_length": 1210.9296875,
|
|
"completions/mean_terminated_length": 853.6432800292969,
|
|
"completions/min_length": 54.5,
|
|
"completions/min_terminated_length": 54.5,
|
|
"entropy": 2.7617363315075636,
|
|
"epoch": 0.024943310657596373,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.00043528827609407017,
|
|
"learning_rate": 9.773242630385487e-07,
|
|
"loss": -0.0082,
|
|
"num_tokens": 2178859.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3079812675714493,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.06099375709891319,
|
|
"step": 11,
|
|
"step_time": 599.1445474587381
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3483.5,
|
|
"completions/mean_length": 1608.015625,
|
|
"completions/mean_terminated_length": 998.5797119140625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.71538058295846,
|
|
"epoch": 0.027210884353741496,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.130859375,
|
|
"kl": 0.0003708188855853223,
|
|
"learning_rate": 9.750566893424036e-07,
|
|
"loss": 0.0254,
|
|
"num_tokens": 2409449.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3062363266944885,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 12,
|
|
"step_time": 604.6724895187654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3624.5,
|
|
"completions/mean_length": 1378.3046875,
|
|
"completions/mean_terminated_length": 844.677490234375,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 3.3661539182066917,
|
|
"epoch": 0.02947845804988662,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.220703125,
|
|
"kl": 0.00040208527752838563,
|
|
"learning_rate": 9.727891156462584e-07,
|
|
"loss": -0.0273,
|
|
"num_tokens": 2609844.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2628627270460129,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 13,
|
|
"step_time": 633.5494570918381
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4007.0,
|
|
"completions/mean_length": 1187.21875,
|
|
"completions/mean_terminated_length": 831.5615844726562,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"entropy": 3.0186092387884855,
|
|
"epoch": 0.031746031746031744,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1943359375,
|
|
"kl": 0.00042435936575202504,
|
|
"learning_rate": 9.705215419501134e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 2786412.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.42632731795310974,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 14,
|
|
"step_time": 591.7922107283957
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3788.5,
|
|
"completions/mean_length": 1438.6640625,
|
|
"completions/mean_terminated_length": 831.2064819335938,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.133930031210184,
|
|
"epoch": 0.034013605442176874,
|
|
"frac_reward_zero_std": 0.96875,
|
|
"grad_norm": 0.080078125,
|
|
"kl": 0.0004019950811198214,
|
|
"learning_rate": 9.682539682539682e-07,
|
|
"loss": -0.0017,
|
|
"num_tokens": 2995009.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.125,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 15,
|
|
"step_time": 606.26203110395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4070.0,
|
|
"completions/mean_length": 1565.796875,
|
|
"completions/mean_terminated_length": 1040.6603698730469,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 4.076895218342543,
|
|
"epoch": 0.036281179138321996,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.15234375,
|
|
"kl": 0.000358647768962328,
|
|
"learning_rate": 9.65986394557823e-07,
|
|
"loss": 0.0219,
|
|
"num_tokens": 3219087.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.17743311822414398,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 16,
|
|
"step_time": 600.7762890821323
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3746.5,
|
|
"completions/mean_length": 1242.6015625,
|
|
"completions/mean_terminated_length": 806.0835876464844,
|
|
"completions/min_length": 42.5,
|
|
"completions/min_terminated_length": 42.5,
|
|
"entropy": 3.07886391505599,
|
|
"epoch": 0.03854875283446712,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.0004230019103488303,
|
|
"learning_rate": 9.63718820861678e-07,
|
|
"loss": 0.0201,
|
|
"num_tokens": 3402980.0,
|
|
"reward": 0.02734375,
|
|
"reward_std": 0.17220696061849594,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 17,
|
|
"step_time": 593.2202473981306
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3938.0,
|
|
"completions/mean_length": 1397.1953125,
|
|
"completions/mean_terminated_length": 978.6914978027344,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 4.573173347860575,
|
|
"epoch": 0.04081632653061224,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.169921875,
|
|
"kl": 0.0003518174189593992,
|
|
"learning_rate": 9.614512471655328e-07,
|
|
"loss": -0.0097,
|
|
"num_tokens": 3604213.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3089591860771179,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 18,
|
|
"step_time": 588.3413797696121
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3858.5,
|
|
"completions/mean_length": 1294.984375,
|
|
"completions/mean_terminated_length": 868.6067504882812,
|
|
"completions/min_length": 59.0,
|
|
"completions/min_terminated_length": 59.0,
|
|
"entropy": 3.730927578173578,
|
|
"epoch": 0.04308390022675737,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"kl": 0.00038500537357322173,
|
|
"learning_rate": 9.591836734693876e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 3795731.0,
|
|
"reward": 0.18359375,
|
|
"reward_std": 0.5490992665290833,
|
|
"rewards/accuracy_reward_func/mean": 0.171875,
|
|
"rewards/accuracy_reward_func/std": 0.5463753789663315,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 19,
|
|
"step_time": 665.4319940893911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3696.5,
|
|
"completions/mean_length": 1467.2109375,
|
|
"completions/mean_terminated_length": 994.596923828125,
|
|
"completions/min_length": 39.0,
|
|
"completions/min_terminated_length": 39.0,
|
|
"entropy": 3.785406796261668,
|
|
"epoch": 0.045351473922902494,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.00038856124820085824,
|
|
"learning_rate": 9.569160997732426e-07,
|
|
"loss": 0.0419,
|
|
"num_tokens": 4007178.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.3037330359220505,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 20,
|
|
"step_time": 592.561503094621
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3836.5,
|
|
"completions/mean_length": 1551.0625,
|
|
"completions/mean_terminated_length": 1051.445556640625,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.296387929469347,
|
|
"epoch": 0.047619047619047616,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1708984375,
|
|
"kl": 0.0003285188749941881,
|
|
"learning_rate": 9.546485260770974e-07,
|
|
"loss": 0.0276,
|
|
"num_tokens": 4230118.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.3057980537414551,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 21,
|
|
"step_time": 619.5003909710795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3641.5,
|
|
"completions/mean_length": 1341.0703125,
|
|
"completions/mean_terminated_length": 892.7926330566406,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 3.226698711514473,
|
|
"epoch": 0.049886621315192746,
|
|
"frac_reward_zero_std": 0.96875,
|
|
"grad_norm": 0.1171875,
|
|
"kl": 0.0004377866116556106,
|
|
"learning_rate": 9.523809523809522e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 4431731.0,
|
|
"reward": 0.01953125,
|
|
"reward_std": 0.0922437533736229,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 22,
|
|
"step_time": 664.7066006427631
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3521.0,
|
|
"completions/mean_length": 1580.8203125,
|
|
"completions/mean_terminated_length": 1011.142578125,
|
|
"completions/min_length": 36.0,
|
|
"completions/min_terminated_length": 36.0,
|
|
"entropy": 3.4629636611789465,
|
|
"epoch": 0.05215419501133787,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.126953125,
|
|
"kl": 0.00039756272417434957,
|
|
"learning_rate": 9.501133786848072e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 4659060.0,
|
|
"reward": 0.02734375,
|
|
"reward_std": 0.17220696061849594,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 23,
|
|
"step_time": 609.2390451589599
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3862.5,
|
|
"completions/mean_length": 1417.40625,
|
|
"completions/mean_terminated_length": 857.968505859375,
|
|
"completions/min_length": 8.5,
|
|
"completions/min_terminated_length": 8.5,
|
|
"entropy": 3.3345618154853582,
|
|
"epoch": 0.05442176870748299,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.00041635158777353354,
|
|
"learning_rate": 9.47845804988662e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 4865088.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.4368836283683777,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.41934312880039215,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 24,
|
|
"step_time": 605.0491937622428
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3432.0,
|
|
"completions/mean_length": 1409.046875,
|
|
"completions/mean_terminated_length": 841.3591918945312,
|
|
"completions/min_length": 8.5,
|
|
"completions/min_terminated_length": 8.5,
|
|
"entropy": 3.0420125983655453,
|
|
"epoch": 0.05668934240362812,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.00044511446731121396,
|
|
"learning_rate": 9.45578231292517e-07,
|
|
"loss": -0.0466,
|
|
"num_tokens": 5070970.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.37423862516880035,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 25,
|
|
"step_time": 617.6828591921367
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4066.5,
|
|
"completions/mean_length": 1433.484375,
|
|
"completions/mean_terminated_length": 909.6171875,
|
|
"completions/min_length": 41.5,
|
|
"completions/min_terminated_length": 41.5,
|
|
"entropy": 3.7978090457618237,
|
|
"epoch": 0.05895691609977324,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.12890625,
|
|
"kl": 0.0003817280889961694,
|
|
"learning_rate": 9.433106575963718e-07,
|
|
"loss": 0.0094,
|
|
"num_tokens": 5277956.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.22862860560417175,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 26,
|
|
"step_time": 593.7618179973215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3716.0,
|
|
"completions/mean_length": 1305.46875,
|
|
"completions/mean_terminated_length": 918.2090148925781,
|
|
"completions/min_length": 94.0,
|
|
"completions/min_terminated_length": 94.0,
|
|
"entropy": 3.155247835442424,
|
|
"epoch": 0.061224489795918366,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.201171875,
|
|
"kl": 0.0004310551134949492,
|
|
"learning_rate": 9.410430839002267e-07,
|
|
"loss": 0.0454,
|
|
"num_tokens": 5467888.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.27729756385087967,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 27,
|
|
"step_time": 595.4528097324073
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4029.5,
|
|
"completions/mean_length": 1341.5078125,
|
|
"completions/mean_terminated_length": 801.0039978027344,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 3.1700076377019286,
|
|
"epoch": 0.06349206349206349,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1552734375,
|
|
"kl": 0.00045847328192394343,
|
|
"learning_rate": 9.387755102040816e-07,
|
|
"loss": 0.0124,
|
|
"num_tokens": 5662897.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3062363266944885,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 28,
|
|
"step_time": 598.0856093554758
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3635.0,
|
|
"completions/mean_length": 1317.609375,
|
|
"completions/mean_terminated_length": 673.3257751464844,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 2.2795989606529474,
|
|
"epoch": 0.06575963718820861,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2470703125,
|
|
"kl": 0.0005228335512583726,
|
|
"learning_rate": 9.365079365079365e-07,
|
|
"loss": -0.07,
|
|
"num_tokens": 5859167.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.34778308868408203,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.29378482699394226,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 29,
|
|
"step_time": 647.6077925832942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3737.0,
|
|
"completions/mean_length": 1378.4921875,
|
|
"completions/mean_terminated_length": 935.5129699707031,
|
|
"completions/min_length": 75.5,
|
|
"completions/min_terminated_length": 75.5,
|
|
"entropy": 3.2670377176254988,
|
|
"epoch": 0.06802721088435375,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1787109375,
|
|
"kl": 0.0004232220730955305,
|
|
"learning_rate": 9.342403628117913e-07,
|
|
"loss": 0.0307,
|
|
"num_tokens": 6060122.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.34078919887542725,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 30,
|
|
"step_time": 605.6695272815414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3844.0,
|
|
"completions/mean_length": 1264.5,
|
|
"completions/mean_terminated_length": 967.5105590820312,
|
|
"completions/min_length": 93.5,
|
|
"completions/min_terminated_length": 93.5,
|
|
"entropy": 4.282577725127339,
|
|
"epoch": 0.07029478458049887,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.0004078878860127588,
|
|
"learning_rate": 9.319727891156463e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 6247282.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.21921012550592422,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 31,
|
|
"step_time": 605.2760956226848
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3761.0,
|
|
"completions/mean_length": 1421.5625,
|
|
"completions/mean_terminated_length": 956.3410034179688,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.9570772480219603,
|
|
"epoch": 0.07256235827664399,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.220703125,
|
|
"kl": 0.00040325274949282175,
|
|
"learning_rate": 9.29705215419501e-07,
|
|
"loss": 0.0311,
|
|
"num_tokens": 6454118.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34598222374916077,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 32,
|
|
"step_time": 594.1900004739873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3791.5,
|
|
"completions/mean_length": 1409.953125,
|
|
"completions/mean_terminated_length": 912.5370483398438,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 3.804439751431346,
|
|
"epoch": 0.07482993197278912,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1376953125,
|
|
"kl": 0.00041808080868577235,
|
|
"learning_rate": 9.274376417233559e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 6657528.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.19257627427577972,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 33,
|
|
"step_time": 597.116535522975
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3800.5,
|
|
"completions/mean_length": 1313.875,
|
|
"completions/mean_terminated_length": 999.6913146972656,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.7660043127834797,
|
|
"epoch": 0.07709750566893424,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2314453125,
|
|
"kl": 0.00040203000435212743,
|
|
"learning_rate": 9.251700680272108e-07,
|
|
"loss": 0.0176,
|
|
"num_tokens": 6848420.0,
|
|
"reward": 0.09765625,
|
|
"reward_std": 0.39702552556991577,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 34,
|
|
"step_time": 588.760639490094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3577.0,
|
|
"completions/mean_length": 1154.796875,
|
|
"completions/mean_terminated_length": 673.73779296875,
|
|
"completions/min_length": 32.0,
|
|
"completions/min_terminated_length": 32.0,
|
|
"entropy": 2.520525285974145,
|
|
"epoch": 0.07936507936507936,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2490234375,
|
|
"kl": 0.00047929613219821476,
|
|
"learning_rate": 9.229024943310657e-07,
|
|
"loss": 0.0117,
|
|
"num_tokens": 7020838.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.30299752950668335,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.27048972249031067,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 35,
|
|
"step_time": 610.717342778109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3509.5,
|
|
"completions/mean_length": 1125.5546875,
|
|
"completions/mean_terminated_length": 701.979736328125,
|
|
"completions/min_length": 51.5,
|
|
"completions/min_terminated_length": 51.5,
|
|
"entropy": 3.0911447647958994,
|
|
"epoch": 0.08163265306122448,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.00045987328167029773,
|
|
"learning_rate": 9.206349206349205e-07,
|
|
"loss": 0.0551,
|
|
"num_tokens": 7187557.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3954896926879883,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 36,
|
|
"step_time": 588.8801189982332
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3893.0,
|
|
"completions/mean_length": 1400.6171875,
|
|
"completions/mean_terminated_length": 980.9603576660156,
|
|
"completions/min_length": 6.5,
|
|
"completions/min_terminated_length": 6.5,
|
|
"entropy": 3.7993264254182577,
|
|
"epoch": 0.08390022675736962,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.0004099325874449278,
|
|
"learning_rate": 9.183673469387755e-07,
|
|
"loss": 0.0228,
|
|
"num_tokens": 7391876.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.21423126757144928,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 37,
|
|
"step_time": 604.1444621784613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3806.0,
|
|
"completions/mean_length": 1468.0390625,
|
|
"completions/mean_terminated_length": 923.6571655273438,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.549352679401636,
|
|
"epoch": 0.08616780045351474,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.00044676465176962665,
|
|
"learning_rate": 9.160997732426303e-07,
|
|
"loss": 0.0222,
|
|
"num_tokens": 7603229.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.3723399490118027,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 38,
|
|
"step_time": 590.1167810624465
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3608.5,
|
|
"completions/mean_length": 1329.921875,
|
|
"completions/mean_terminated_length": 794.9758911132812,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 3.4886126536875963,
|
|
"epoch": 0.08843537414965986,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.00046471107089018915,
|
|
"learning_rate": 9.138321995464853e-07,
|
|
"loss": -0.0098,
|
|
"num_tokens": 7799027.0,
|
|
"reward": 0.14453125,
|
|
"reward_std": 0.4880780875682831,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48353175818920135,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 39,
|
|
"step_time": 606.6144362050109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3697.5,
|
|
"completions/mean_length": 1280.03125,
|
|
"completions/mean_terminated_length": 839.4837646484375,
|
|
"completions/min_length": 20.5,
|
|
"completions/min_terminated_length": 20.5,
|
|
"entropy": 3.654280535876751,
|
|
"epoch": 0.09070294784580499,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.12353515625,
|
|
"kl": 0.00046478183367071324,
|
|
"learning_rate": 9.115646258503401e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 7985183.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.3003681004047394,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 40,
|
|
"step_time": 589.6266784924082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2835.5,
|
|
"completions/mean_length": 1271.2890625,
|
|
"completions/mean_terminated_length": 773.3348693847656,
|
|
"completions/min_length": 85.5,
|
|
"completions/min_terminated_length": 85.5,
|
|
"entropy": 2.9861853010952473,
|
|
"epoch": 0.09297052154195011,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2265625,
|
|
"kl": 0.00046541515530407196,
|
|
"learning_rate": 9.09297052154195e-07,
|
|
"loss": 0.0296,
|
|
"num_tokens": 8172404.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.3428210914134979,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 41,
|
|
"step_time": 592.6969041889533
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3665.5,
|
|
"completions/mean_length": 1196.203125,
|
|
"completions/mean_terminated_length": 950.4576416015625,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 3.5352232959121466,
|
|
"epoch": 0.09523809523809523,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.00044147984135634033,
|
|
"learning_rate": 9.070294784580499e-07,
|
|
"loss": 0.0382,
|
|
"num_tokens": 8348714.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4056457281112671,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.39548972249031067,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 42,
|
|
"step_time": 599.7052725390531
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3302.5,
|
|
"completions/mean_length": 1222.3828125,
|
|
"completions/mean_terminated_length": 783.2756958007812,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.064804270863533,
|
|
"epoch": 0.09750566893424037,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.251953125,
|
|
"kl": 0.0004974609882992809,
|
|
"learning_rate": 9.047619047619047e-07,
|
|
"loss": -0.0257,
|
|
"num_tokens": 8529679.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31388580799102783,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 43,
|
|
"step_time": 605.891956679523
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3939.0,
|
|
"completions/mean_length": 1661.859375,
|
|
"completions/mean_terminated_length": 1100.1346435546875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.5304269678890705,
|
|
"epoch": 0.09977324263038549,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.0004575031762215076,
|
|
"learning_rate": 9.024943310657595e-07,
|
|
"loss": 0.0182,
|
|
"num_tokens": 8766585.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.4579998552799225,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 44,
|
|
"step_time": 609.272553737741
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3914.0,
|
|
"completions/mean_length": 1193.4296875,
|
|
"completions/mean_terminated_length": 894.8565368652344,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.180556582286954,
|
|
"epoch": 0.10204081632653061,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1044921875,
|
|
"kl": 0.0004615836469383794,
|
|
"learning_rate": 9.002267573696145e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 8943480.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.21921012550592422,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 45,
|
|
"step_time": 606.0064869229682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3297.0,
|
|
"completions/mean_length": 1318.296875,
|
|
"completions/mean_terminated_length": 836.4284362792969,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 3.324997106567025,
|
|
"epoch": 0.10430839002267574,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.171875,
|
|
"kl": 0.0004851645394410298,
|
|
"learning_rate": 8.979591836734693e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 9135110.0,
|
|
"reward": 0.01171875,
|
|
"reward_std": 0.07509202510118484,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 46,
|
|
"step_time": 591.3534867856652
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3921.5,
|
|
"completions/mean_length": 1362.703125,
|
|
"completions/mean_terminated_length": 887.37744140625,
|
|
"completions/min_length": 48.0,
|
|
"completions/min_terminated_length": 48.0,
|
|
"entropy": 3.4120216239243746,
|
|
"epoch": 0.10657596371882086,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.22265625,
|
|
"kl": 0.000441622027210542,
|
|
"learning_rate": 8.956916099773242e-07,
|
|
"loss": -0.0113,
|
|
"num_tokens": 9332408.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.3428210914134979,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 47,
|
|
"step_time": 592.4939067498781
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3773.5,
|
|
"completions/mean_length": 1337.7265625,
|
|
"completions/mean_terminated_length": 889.4365234375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 3.588590629398823,
|
|
"epoch": 0.10884353741496598,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1806640625,
|
|
"kl": 0.0004888545086032536,
|
|
"learning_rate": 8.934240362811791e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 9528173.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.3428210914134979,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 48,
|
|
"step_time": 595.2394825532101
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3751.5,
|
|
"completions/mean_length": 1370.3125,
|
|
"completions/mean_terminated_length": 981.296630859375,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 3.3204429261386395,
|
|
"epoch": 0.1111111111111111,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1201171875,
|
|
"kl": 0.0004708280312115676,
|
|
"learning_rate": 8.91156462585034e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 9727061.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08451050519943237,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 49,
|
|
"step_time": 619.0451905168593
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3857.0,
|
|
"completions/mean_length": 1159.9296875,
|
|
"completions/mean_terminated_length": 856.0658874511719,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.274019964039326,
|
|
"epoch": 0.11337868480725624,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1904296875,
|
|
"kl": 0.00045126326358513325,
|
|
"learning_rate": 8.888888888888888e-07,
|
|
"loss": 0.025,
|
|
"num_tokens": 9899344.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.26565735042095184,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 50,
|
|
"step_time": 594.9196514263749
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3712.0,
|
|
"completions/mean_length": 1539.75,
|
|
"completions/mean_terminated_length": 1009.2075805664062,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.347749046981335,
|
|
"epoch": 0.11564625850340136,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.212890625,
|
|
"kl": 0.0004338094959166483,
|
|
"learning_rate": 8.866213151927438e-07,
|
|
"loss": 0.0298,
|
|
"num_tokens": 10119776.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31276239454746246,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 51,
|
|
"step_time": 604.532951598987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2861.5,
|
|
"completions/mean_length": 1311.625,
|
|
"completions/mean_terminated_length": 793.7900390625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 3.2434661109000444,
|
|
"epoch": 0.11791383219954649,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.0004534392055575154,
|
|
"learning_rate": 8.843537414965985e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 10311152.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.31276239454746246,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 52,
|
|
"step_time": 606.0263563157059
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3860.0,
|
|
"completions/mean_length": 1324.3984375,
|
|
"completions/mean_terminated_length": 869.1448669433594,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 3.9168042186647654,
|
|
"epoch": 0.12018140589569161,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1103515625,
|
|
"kl": 0.0004939305722473364,
|
|
"learning_rate": 8.820861678004535e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 10502727.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.16884202510118484,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 53,
|
|
"step_time": 588.4414443285204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3656.5,
|
|
"completions/mean_length": 1435.96875,
|
|
"completions/mean_terminated_length": 885.0552673339844,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.6197192538529634,
|
|
"epoch": 0.12244897959183673,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.201171875,
|
|
"kl": 0.0004839639314013766,
|
|
"learning_rate": 8.798185941043083e-07,
|
|
"loss": -0.054,
|
|
"num_tokens": 10712659.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.3572239279747009,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 54,
|
|
"step_time": 610.2372101433575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3640.5,
|
|
"completions/mean_length": 1367.390625,
|
|
"completions/mean_terminated_length": 960.486328125,
|
|
"completions/min_length": 58.5,
|
|
"completions/min_terminated_length": 58.5,
|
|
"entropy": 3.9621335081756115,
|
|
"epoch": 0.12471655328798185,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.19921875,
|
|
"kl": 0.00046764538933530275,
|
|
"learning_rate": 8.775510204081632e-07,
|
|
"loss": 0.0121,
|
|
"num_tokens": 10911001.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.18162544071674347,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 55,
|
|
"step_time": 596.5860891141929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3507.5,
|
|
"completions/mean_length": 1238.6640625,
|
|
"completions/mean_terminated_length": 770.5470275878906,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 2.911743665114045,
|
|
"epoch": 0.12698412698412698,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.0005278955277390196,
|
|
"learning_rate": 8.752834467120181e-07,
|
|
"loss": 0.0388,
|
|
"num_tokens": 11093198.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.3057980537414551,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 56,
|
|
"step_time": 602.7431881749071
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3881.5,
|
|
"completions/mean_length": 1367.5234375,
|
|
"completions/mean_terminated_length": 978.820556640625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.401599169708788,
|
|
"epoch": 0.1292517006802721,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.0004408286617945123,
|
|
"learning_rate": 8.73015873015873e-07,
|
|
"loss": 0.029,
|
|
"num_tokens": 11293209.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5637782216072083,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.527618482708931,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.06099375709891319,
|
|
"step": 57,
|
|
"step_time": 629.6106900847517
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3375.0,
|
|
"completions/mean_length": 1267.1171875,
|
|
"completions/mean_terminated_length": 834.4556884765625,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 2.8903925102204084,
|
|
"epoch": 0.13151927437641722,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.1494140625,
|
|
"kl": 0.0004891294333901897,
|
|
"learning_rate": 8.707482993197278e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 11477936.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 58,
|
|
"step_time": 591.5403750715777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3903.5,
|
|
"completions/mean_length": 1221.6484375,
|
|
"completions/mean_terminated_length": 867.8571472167969,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.4592435099184513,
|
|
"epoch": 0.13378684807256236,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.19921875,
|
|
"kl": 0.00048117998039742815,
|
|
"learning_rate": 8.684807256235828e-07,
|
|
"loss": -0.0208,
|
|
"num_tokens": 11662035.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.3089591860771179,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 59,
|
|
"step_time": 617.3197298264131
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3738.0,
|
|
"completions/mean_length": 1492.890625,
|
|
"completions/mean_terminated_length": 1009.9540405273438,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.7502216193825006,
|
|
"epoch": 0.1360544217687075,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.0004347923636487394,
|
|
"learning_rate": 8.662131519274376e-07,
|
|
"loss": 0.0369,
|
|
"num_tokens": 11875833.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 60,
|
|
"step_time": 593.7963156336918
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3947.5,
|
|
"completions/mean_length": 1247.8125,
|
|
"completions/mean_terminated_length": 948.7422790527344,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.10028101503849,
|
|
"epoch": 0.1383219954648526,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2001953125,
|
|
"kl": 0.0005185582331250771,
|
|
"learning_rate": 8.639455782312925e-07,
|
|
"loss": 0.0383,
|
|
"num_tokens": 12058597.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34415413439273834,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 61,
|
|
"step_time": 595.987687363755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3756.5,
|
|
"completions/mean_length": 1450.0625,
|
|
"completions/mean_terminated_length": 811.7161560058594,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 3.791416520252824,
|
|
"epoch": 0.14058956916099774,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.216796875,
|
|
"kl": 0.000482378878587042,
|
|
"learning_rate": 8.616780045351474e-07,
|
|
"loss": 0.0598,
|
|
"num_tokens": 12270933.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.29821833968162537,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 62,
|
|
"step_time": 639.7064624852501
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3913.0,
|
|
"completions/mean_length": 1566.1171875,
|
|
"completions/mean_terminated_length": 1041.0471801757812,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 4.66802628710866,
|
|
"epoch": 0.14285714285714285,
|
|
"frac_reward_zero_std": 0.96875,
|
|
"grad_norm": 0.06103515625,
|
|
"kl": 0.00040643151942276745,
|
|
"learning_rate": 8.594104308390022e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 12495768.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.17536810040473938,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 63,
|
|
"step_time": 597.4973760107532
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3819.0,
|
|
"completions/mean_length": 1186.453125,
|
|
"completions/mean_terminated_length": 799.9868774414062,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 3.6843310575932264,
|
|
"epoch": 0.14512471655328799,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.0004871052587986924,
|
|
"learning_rate": 8.57142857142857e-07,
|
|
"loss": 0.0321,
|
|
"num_tokens": 12671826.0,
|
|
"reward": 0.12109375,
|
|
"reward_std": 0.4517260193824768,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.44585782289505005,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 64,
|
|
"step_time": 607.9777935571037
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3789.5,
|
|
"completions/mean_length": 1403.6953125,
|
|
"completions/mean_terminated_length": 992.7754821777344,
|
|
"completions/min_length": 5.5,
|
|
"completions/min_terminated_length": 5.5,
|
|
"entropy": 4.1216657515615225,
|
|
"epoch": 0.1473922902494331,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2177734375,
|
|
"kl": 0.0004439708786776464,
|
|
"learning_rate": 8.54875283446712e-07,
|
|
"loss": 0.0501,
|
|
"num_tokens": 12874711.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3109140843153,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 65,
|
|
"step_time": 610.7431227061898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3714.0,
|
|
"completions/mean_length": 1354.2421875,
|
|
"completions/mean_terminated_length": 876.5531005859375,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"entropy": 3.687948001548648,
|
|
"epoch": 0.14965986394557823,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2080078125,
|
|
"kl": 0.0004651461445064342,
|
|
"learning_rate": 8.526077097505668e-07,
|
|
"loss": 0.0343,
|
|
"num_tokens": 13072098.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.2660238593816757,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 66,
|
|
"step_time": 593.1527040861547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4051.5,
|
|
"completions/mean_length": 1323.0625,
|
|
"completions/mean_terminated_length": 873.1241455078125,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"entropy": 4.224043572321534,
|
|
"epoch": 0.15192743764172337,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.193359375,
|
|
"kl": 0.0005088694788355497,
|
|
"learning_rate": 8.503401360544217e-07,
|
|
"loss": -0.0178,
|
|
"num_tokens": 13266110.0,
|
|
"reward": 0.08203125,
|
|
"reward_std": 0.3517485707998276,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09224375709891319,
|
|
"step": 67,
|
|
"step_time": 604.5064916228876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3337.5,
|
|
"completions/mean_length": 1203.515625,
|
|
"completions/mean_terminated_length": 848.2982482910156,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 3.2738924492150545,
|
|
"epoch": 0.15419501133786848,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.287109375,
|
|
"kl": 0.0005158751378075976,
|
|
"learning_rate": 8.480725623582766e-07,
|
|
"loss": 0.0422,
|
|
"num_tokens": 13444504.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.4579998552799225,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.45701706409454346,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 68,
|
|
"step_time": 602.0630125850439
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1328125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3615.5,
|
|
"completions/mean_length": 1392.4921875,
|
|
"completions/mean_terminated_length": 976.9803771972656,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 4.119066654704511,
|
|
"epoch": 0.1564625850340136,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.00045541685312855407,
|
|
"learning_rate": 8.458049886621315e-07,
|
|
"loss": -0.0348,
|
|
"num_tokens": 13645543.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.29378482699394226,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 69,
|
|
"step_time": 593.0577980796807
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3935.0,
|
|
"completions/mean_length": 1399.390625,
|
|
"completions/mean_terminated_length": 933.7881469726562,
|
|
"completions/min_length": 129.5,
|
|
"completions/min_terminated_length": 129.5,
|
|
"entropy": 3.355590634047985,
|
|
"epoch": 0.15873015873015872,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1806640625,
|
|
"kl": 0.0005192129283386748,
|
|
"learning_rate": 8.435374149659864e-07,
|
|
"loss": 0.0147,
|
|
"num_tokens": 13848217.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.21248632669448853,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 70,
|
|
"step_time": 599.6777586075477
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3537.0,
|
|
"completions/mean_length": 1360.4765625,
|
|
"completions/mean_terminated_length": 853.8981628417969,
|
|
"completions/min_length": 9.5,
|
|
"completions/min_terminated_length": 9.5,
|
|
"entropy": 3.191288085654378,
|
|
"epoch": 0.16099773242630386,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1748046875,
|
|
"kl": 0.0005019806021664408,
|
|
"learning_rate": 8.412698412698413e-07,
|
|
"loss": -0.0037,
|
|
"num_tokens": 14046134.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34415413439273834,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 71,
|
|
"step_time": 599.8396325754002
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.21875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3744.0,
|
|
"completions/mean_length": 1680.890625,
|
|
"completions/mean_terminated_length": 1004.6346435546875,
|
|
"completions/min_length": 50.0,
|
|
"completions/min_terminated_length": 50.0,
|
|
"entropy": 4.126177066937089,
|
|
"epoch": 0.16326530612244897,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1513671875,
|
|
"kl": 0.0005027235952184128,
|
|
"learning_rate": 8.39002267573696e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 14285372.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3408081382513046,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 72,
|
|
"step_time": 594.7445488786325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3948.5,
|
|
"completions/mean_length": 1401.875,
|
|
"completions/mean_terminated_length": 934.7810363769531,
|
|
"completions/min_length": 4.5,
|
|
"completions/min_terminated_length": 4.5,
|
|
"entropy": 4.1191865261644125,
|
|
"epoch": 0.1655328798185941,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.115234375,
|
|
"kl": 0.0004872900472037145,
|
|
"learning_rate": 8.36734693877551e-07,
|
|
"loss": 0.0228,
|
|
"num_tokens": 14490792.0,
|
|
"reward": 0.02734375,
|
|
"reward_std": 0.19010721147060394,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 73,
|
|
"step_time": 605.806323479861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3813.0,
|
|
"completions/mean_length": 1314.6796875,
|
|
"completions/mean_terminated_length": 859.5545349121094,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 3.5274351220577955,
|
|
"epoch": 0.16780045351473924,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.232421875,
|
|
"kl": 0.0005006528513149533,
|
|
"learning_rate": 8.344671201814058e-07,
|
|
"loss": 0.0303,
|
|
"num_tokens": 14684063.0,
|
|
"reward": 0.18359375,
|
|
"reward_std": 0.5659296214580536,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.527618482708931,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 74,
|
|
"step_time": 601.1992794224061
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4038.0,
|
|
"completions/mean_length": 1398.1875,
|
|
"completions/mean_terminated_length": 953.0533142089844,
|
|
"completions/min_length": 15.5,
|
|
"completions/min_terminated_length": 15.5,
|
|
"entropy": 3.780844986438751,
|
|
"epoch": 0.17006802721088435,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.0005055789238213038,
|
|
"learning_rate": 8.321995464852607e-07,
|
|
"loss": 0.0167,
|
|
"num_tokens": 14887503.0,
|
|
"reward": 0.06640625,
|
|
"reward_std": 0.3414069563150406,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 75,
|
|
"step_time": 594.4315239707939
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3696.5,
|
|
"completions/mean_length": 1401.9609375,
|
|
"completions/mean_terminated_length": 932.441650390625,
|
|
"completions/min_length": 96.5,
|
|
"completions/min_terminated_length": 96.5,
|
|
"entropy": 4.313092632219195,
|
|
"epoch": 0.17233560090702948,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.23046875,
|
|
"kl": 0.0005012169604015071,
|
|
"learning_rate": 8.299319727891156e-07,
|
|
"loss": 0.0375,
|
|
"num_tokens": 15089758.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.3109140843153,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 76,
|
|
"step_time": 590.3546146731824
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3951.5,
|
|
"completions/mean_length": 1343.8671875,
|
|
"completions/mean_terminated_length": 1005.885986328125,
|
|
"completions/min_length": 20.5,
|
|
"completions/min_terminated_length": 20.5,
|
|
"entropy": 3.6281288098543882,
|
|
"epoch": 0.1746031746031746,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.275390625,
|
|
"kl": 0.00048217516268778127,
|
|
"learning_rate": 8.276643990929705e-07,
|
|
"loss": 0.0711,
|
|
"num_tokens": 15289129.0,
|
|
"reward": 0.15234375,
|
|
"reward_std": 0.4942307323217392,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 77,
|
|
"step_time": 629.5171679407358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3971.5,
|
|
"completions/mean_length": 1422.21875,
|
|
"completions/mean_terminated_length": 1040.2500610351562,
|
|
"completions/min_length": 63.5,
|
|
"completions/min_terminated_length": 63.5,
|
|
"entropy": 4.338170548900962,
|
|
"epoch": 0.17687074829931973,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.2041015625,
|
|
"kl": 0.0004632392774510663,
|
|
"learning_rate": 8.253968253968253e-07,
|
|
"loss": 0.0613,
|
|
"num_tokens": 15494701.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42883121967315674,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 78,
|
|
"step_time": 593.3397142631002
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.109375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3159.0,
|
|
"completions/mean_length": 1178.703125,
|
|
"completions/mean_terminated_length": 819.6268615722656,
|
|
"completions/min_length": 82.5,
|
|
"completions/min_terminated_length": 82.5,
|
|
"entropy": 3.7087379712611437,
|
|
"epoch": 0.17913832199546487,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.169921875,
|
|
"kl": 0.0005126559735799674,
|
|
"learning_rate": 8.231292517006803e-07,
|
|
"loss": 0.0204,
|
|
"num_tokens": 15669327.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.25688404589891434,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 79,
|
|
"step_time": 595.9942190372385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4061.0,
|
|
"completions/mean_length": 1321.203125,
|
|
"completions/mean_terminated_length": 924.8036193847656,
|
|
"completions/min_length": 81.0,
|
|
"completions/min_terminated_length": 81.0,
|
|
"entropy": 4.271406231448054,
|
|
"epoch": 0.18140589569160998,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.00046873624796717195,
|
|
"learning_rate": 8.208616780045351e-07,
|
|
"loss": 0.0269,
|
|
"num_tokens": 15861613.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.20661810040473938,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 80,
|
|
"step_time": 592.3808848424815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3996.5,
|
|
"completions/mean_length": 1224.046875,
|
|
"completions/mean_terminated_length": 953.5816040039062,
|
|
"completions/min_length": 96.0,
|
|
"completions/min_terminated_length": 96.0,
|
|
"entropy": 3.661328922957182,
|
|
"epoch": 0.1836734693877551,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.00046417086150540854,
|
|
"learning_rate": 8.1859410430839e-07,
|
|
"loss": 0.0969,
|
|
"num_tokens": 16040943.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5218297988176346,
|
|
"rewards/accuracy_reward_func/mean": 0.140625,
|
|
"rewards/accuracy_reward_func/std": 0.506826862692833,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 81,
|
|
"step_time": 591.166427501943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3809.5,
|
|
"completions/mean_length": 1466.0546875,
|
|
"completions/mean_terminated_length": 1007.4424133300781,
|
|
"completions/min_length": 47.5,
|
|
"completions/min_terminated_length": 47.5,
|
|
"entropy": 4.402091367170215,
|
|
"epoch": 0.18594104308390022,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2099609375,
|
|
"kl": 0.0004720119945886836,
|
|
"learning_rate": 8.163265306122449e-07,
|
|
"loss": 0.0212,
|
|
"num_tokens": 16252866.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.31193090975284576,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 82,
|
|
"step_time": 601.9541582926176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3953.5,
|
|
"completions/mean_length": 1354.015625,
|
|
"completions/mean_terminated_length": 847.1114807128906,
|
|
"completions/min_length": 30.0,
|
|
"completions/min_terminated_length": 30.0,
|
|
"entropy": 3.4956730902194977,
|
|
"epoch": 0.18820861678004536,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.0005511523636414495,
|
|
"learning_rate": 8.140589569160997e-07,
|
|
"loss": -0.0169,
|
|
"num_tokens": 16450664.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.3938441723585129,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 83,
|
|
"step_time": 600.6701087602414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3605.0,
|
|
"completions/mean_length": 1505.0234375,
|
|
"completions/mean_terminated_length": 1057.6773681640625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 4.234385251998901,
|
|
"epoch": 0.19047619047619047,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1640625,
|
|
"kl": 0.00045168545375418034,
|
|
"learning_rate": 8.117913832199546e-07,
|
|
"loss": 0.016,
|
|
"num_tokens": 16666115.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.25652606785297394,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 84,
|
|
"step_time": 587.7375784683973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3895.5,
|
|
"completions/mean_length": 1411.1875,
|
|
"completions/mean_terminated_length": 971.8545227050781,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 3.7613945519551635,
|
|
"epoch": 0.1927437641723356,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.000517658699209278,
|
|
"learning_rate": 8.095238095238095e-07,
|
|
"loss": -0.0027,
|
|
"num_tokens": 16871639.0,
|
|
"reward": 0.05859375,
|
|
"reward_std": 0.30986589193344116,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 85,
|
|
"step_time": 604.7108815400861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3775.0,
|
|
"completions/mean_length": 1438.3203125,
|
|
"completions/mean_terminated_length": 1085.2908935546875,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.597478030249476,
|
|
"epoch": 0.19501133786848074,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.2353515625,
|
|
"kl": 0.0004401639350817277,
|
|
"learning_rate": 8.072562358276643e-07,
|
|
"loss": 0.0232,
|
|
"num_tokens": 17079948.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 86,
|
|
"step_time": 594.5786706414074
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3547.0,
|
|
"completions/mean_length": 1463.40625,
|
|
"completions/mean_terminated_length": 885.6181335449219,
|
|
"completions/min_length": 42.5,
|
|
"completions/min_terminated_length": 42.5,
|
|
"entropy": 3.2632428109645844,
|
|
"epoch": 0.19727891156462585,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.09521484375,
|
|
"kl": 0.0005363671834857087,
|
|
"learning_rate": 8.049886621315193e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 17291036.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 87,
|
|
"step_time": 591.2662706999108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3408.0,
|
|
"completions/mean_length": 1145.75,
|
|
"completions/mean_terminated_length": 748.8887329101562,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 2.831279480829835,
|
|
"epoch": 0.19954648526077098,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.0006093931369832717,
|
|
"learning_rate": 8.027210884353741e-07,
|
|
"loss": -0.0124,
|
|
"num_tokens": 17464188.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3244084417819977,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.27048972249031067,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 88,
|
|
"step_time": 648.395480513107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1796875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3822.0,
|
|
"completions/mean_length": 1535.40625,
|
|
"completions/mean_terminated_length": 975.2217102050781,
|
|
"completions/min_length": 3.5,
|
|
"completions/min_terminated_length": 3.5,
|
|
"entropy": 4.225502125918865,
|
|
"epoch": 0.2018140589569161,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.234375,
|
|
"kl": 0.0004882277917204192,
|
|
"learning_rate": 8.00453514739229e-07,
|
|
"loss": 0.0023,
|
|
"num_tokens": 17685020.0,
|
|
"reward": 0.10546875,
|
|
"reward_std": 0.42434585094451904,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.41934312880039215,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 89,
|
|
"step_time": 617.4666036618873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3873.5,
|
|
"completions/mean_length": 1306.28125,
|
|
"completions/mean_terminated_length": 991.5018310546875,
|
|
"completions/min_length": 69.5,
|
|
"completions/min_terminated_length": 69.5,
|
|
"entropy": 4.6000791024416685,
|
|
"epoch": 0.20408163265306123,
|
|
"frac_reward_zero_std": 0.71875,
|
|
"grad_norm": 0.345703125,
|
|
"kl": 0.0004467499857128132,
|
|
"learning_rate": 7.981859410430839e-07,
|
|
"loss": 0.0685,
|
|
"num_tokens": 17874960.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6037300229072571,
|
|
"rewards/accuracy_reward_func/mean": 0.203125,
|
|
"rewards/accuracy_reward_func/std": 0.6038230657577515,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 90,
|
|
"step_time": 604.7552698785439
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4045.0,
|
|
"completions/mean_length": 1351.59375,
|
|
"completions/mean_terminated_length": 843.3703918457031,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.1051435321569443,
|
|
"epoch": 0.20634920634920634,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.189453125,
|
|
"kl": 0.0005171954862817074,
|
|
"learning_rate": 7.959183673469388e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 18072192.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.21445615589618683,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.21304203569889069,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 91,
|
|
"step_time": 600.6844751848839
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4009.5,
|
|
"completions/mean_length": 1182.4375,
|
|
"completions/mean_terminated_length": 914.9233093261719,
|
|
"completions/min_length": 43.5,
|
|
"completions/min_terminated_length": 43.5,
|
|
"entropy": 3.176400328055024,
|
|
"epoch": 0.20861678004535147,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.228515625,
|
|
"kl": 0.0006316097587841796,
|
|
"learning_rate": 7.936507936507936e-07,
|
|
"loss": 0.0713,
|
|
"num_tokens": 18247568.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34415413439273834,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 92,
|
|
"step_time": 600.9037633007392
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3857.5,
|
|
"completions/mean_length": 1264.0234375,
|
|
"completions/mean_terminated_length": 885.5155334472656,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 3.3863612934947014,
|
|
"epoch": 0.2108843537414966,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.1533203125,
|
|
"kl": 0.0005828815615132044,
|
|
"learning_rate": 7.913832199546485e-07,
|
|
"loss": 0.0275,
|
|
"num_tokens": 18432431.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.20661810040473938,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 93,
|
|
"step_time": 588.9628076292574
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3825.5,
|
|
"completions/mean_length": 1464.2578125,
|
|
"completions/mean_terminated_length": 920.7213745117188,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 3.7992858309298754,
|
|
"epoch": 0.21315192743764172,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1748046875,
|
|
"kl": 0.0004975047263542365,
|
|
"learning_rate": 7.891156462585033e-07,
|
|
"loss": 0.0563,
|
|
"num_tokens": 18642508.0,
|
|
"reward": 0.05078125,
|
|
"reward_std": 0.3037330359220505,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 94,
|
|
"step_time": 590.199854795821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.203125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3217.0,
|
|
"completions/mean_length": 1518.265625,
|
|
"completions/mean_terminated_length": 873.223388671875,
|
|
"completions/min_length": 34.0,
|
|
"completions/min_terminated_length": 34.0,
|
|
"entropy": 3.0404637344181538,
|
|
"epoch": 0.21541950113378686,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2001953125,
|
|
"kl": 0.0006586737342786364,
|
|
"learning_rate": 7.868480725623582e-07,
|
|
"loss": 0.0262,
|
|
"num_tokens": 18861754.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.37508824467658997,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.36897502839565277,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 95,
|
|
"step_time": 600.110039126128
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3854.0,
|
|
"completions/mean_length": 1147.1640625,
|
|
"completions/mean_terminated_length": 814.8431091308594,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"entropy": 3.5139937046915293,
|
|
"epoch": 0.21768707482993196,
|
|
"frac_reward_zero_std": 0.6875,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.0006208947957020428,
|
|
"learning_rate": 7.845804988662131e-07,
|
|
"loss": 0.0483,
|
|
"num_tokens": 19033075.0,
|
|
"reward": 0.18359375,
|
|
"reward_std": 0.5316187739372253,
|
|
"rewards/accuracy_reward_func/mean": 0.171875,
|
|
"rewards/accuracy_reward_func/std": 0.5257505476474762,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.05326050892472267,
|
|
"step": 96,
|
|
"step_time": 612.0829366515391
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3766.0,
|
|
"completions/mean_length": 1298.578125,
|
|
"completions/mean_terminated_length": 937.0536499023438,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 4.03733023442328,
|
|
"epoch": 0.2199546485260771,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.1826171875,
|
|
"kl": 0.0005434328236333386,
|
|
"learning_rate": 7.82312925170068e-07,
|
|
"loss": 0.0332,
|
|
"num_tokens": 19225345.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.2878170385956764,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.043842025101184845,
|
|
"step": 97,
|
|
"step_time": 620.4109515491873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3946.5,
|
|
"completions/mean_length": 1291.34375,
|
|
"completions/mean_terminated_length": 1162.2857360839844,
|
|
"completions/min_length": 64.5,
|
|
"completions/min_terminated_length": 64.5,
|
|
"entropy": 4.210914688184857,
|
|
"epoch": 0.2222222222222222,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.36328125,
|
|
"kl": 0.0004791390597347345,
|
|
"learning_rate": 7.800453514739229e-07,
|
|
"loss": -0.0068,
|
|
"num_tokens": 19414265.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.34598222374916077,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.3380420356988907,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 98,
|
|
"step_time": 587.3077208646573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3505.5,
|
|
"completions/mean_length": 1350.390625,
|
|
"completions/mean_terminated_length": 900.6047668457031,
|
|
"completions/min_length": 147.5,
|
|
"completions/min_terminated_length": 147.5,
|
|
"entropy": 3.5034069549292326,
|
|
"epoch": 0.22448979591836735,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.240234375,
|
|
"kl": 0.0005506929642251635,
|
|
"learning_rate": 7.777777777777778e-07,
|
|
"loss": -0.0211,
|
|
"num_tokens": 19610999.0,
|
|
"reward": 0.07421875,
|
|
"reward_std": 0.3185287415981293,
|
|
"rewards/accuracy_reward_func/mean": 0.046875,
|
|
"rewards/accuracy_reward_func/std": 0.3003681004047394,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 99,
|
|
"step_time": 600.8466322016902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4064.0,
|
|
"completions/mean_length": 1142.0703125,
|
|
"completions/mean_terminated_length": 867.5115661621094,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 3.881171302869916,
|
|
"epoch": 0.22675736961451248,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.205078125,
|
|
"kl": 0.0005395735774982313,
|
|
"learning_rate": 7.755102040816326e-07,
|
|
"loss": 0.011,
|
|
"num_tokens": 19779508.0,
|
|
"reward": 0.03515625,
|
|
"reward_std": 0.20516829937696457,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
|
"rewards/format_reward_func/std": 0.09710253402590752,
|
|
"step": 100,
|
|
"step_time": 591.9800079758279
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3986.5,
|
|
"completions/mean_length": 1238.7421875,
|
|
"completions/mean_terminated_length": 945.7524108886719,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.6473589222878218,
|
|
"epoch": 0.2290249433106576,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.0005329903901838406,
|
|
"learning_rate": 7.732426303854876e-07,
|
|
"loss": 0.0061,
|
|
"num_tokens": 19963339.0,
|
|
"reward": 0.08984375,
|
|
"reward_std": 0.31631264090538025,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.24397502839565277,
|
|
"rewards/format_reward_func/mean": 0.02734375,
|
|
"rewards/format_reward_func/std": 0.11425426602363586,
|
|
"step": 101,
|
|
"step_time": 604.477686543949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3731.5,
|
|
"completions/mean_length": 1222.5078125,
|
|
"completions/mean_terminated_length": 839.1725769042969,
|
|
"completions/min_length": 63.5,
|
|
"completions/min_terminated_length": 63.5,
|
|
"entropy": 3.7527093160897493,
|
|
"epoch": 0.23129251700680273,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2314453125,
|
|
"kl": 0.0005475037442010944,
|
|
"learning_rate": 7.709750566893424e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 20143212.0,
|
|
"reward": 0.04296875,
|
|
"reward_std": 0.22127514332532883,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.17536810040473938,
|
|
"rewards/format_reward_func/mean": 0.01171875,
|
|
"rewards/format_reward_func/std": 0.07509202510118484,
|
|
"step": 102,
|
|
"step_time": 592.505566266831
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4022.0,
|
|
"completions/mean_length": 1107.6328125,
|
|
"completions/mean_terminated_length": 798.4913635253906,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 3.091963429003954,
|
|
"epoch": 0.23356009070294784,
|
|
"frac_reward_zero_std": 0.78125,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.0005909597002755618,
|
|
"learning_rate": 7.687074829931972e-07,
|
|
"loss": -0.0608,
|
|
"num_tokens": 20309593.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4511355459690094,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.44585782289505005,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 103,
|
|
"step_time": 615.5466353255324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3392.5,
|
|
"completions/mean_length": 1189.453125,
|
|
"completions/mean_terminated_length": 710.3790283203125,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 2.8702713698148727,
|
|
"epoch": 0.23582766439909297,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.146484375,
|
|
"kl": 0.0005800623353025003,
|
|
"learning_rate": 7.664399092970521e-07,
|
|
"loss": 0.0195,
|
|
"num_tokens": 20485631.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.4352022409439087,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.06099375709891319,
|
|
"step": 104,
|
|
"step_time": 596.6378222648054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.140625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3404.5,
|
|
"completions/mean_length": 1230.7265625,
|
|
"completions/mean_terminated_length": 762.2291870117188,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 3.1164721120148897,
|
|
"epoch": 0.23809523809523808,
|
|
"frac_reward_zero_std": 0.90625,
|
|
"grad_norm": 0.2294921875,
|
|
"kl": 0.0006275299128901679,
|
|
"learning_rate": 7.64172335600907e-07,
|
|
"loss": 0.0481,
|
|
"num_tokens": 20667916.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.16574779152870178,
|
|
"rewards/accuracy_reward_func/mean": 0.015625,
|
|
"rewards/accuracy_reward_func/std": 0.125,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 105,
|
|
"step_time": 595.2047035517171
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1640625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3978.0,
|
|
"completions/mean_length": 1424.9453125,
|
|
"completions/mean_terminated_length": 901.153564453125,
|
|
"completions/min_length": 44.5,
|
|
"completions/min_terminated_length": 44.5,
|
|
"entropy": 3.49599988758564,
|
|
"epoch": 0.24036281179138322,
|
|
"frac_reward_zero_std": 0.9375,
|
|
"grad_norm": 0.11865234375,
|
|
"kl": 0.0006155359710646735,
|
|
"learning_rate": 7.619047619047618e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 20873273.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08451050519943237,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08451050892472267,
|
|
"step": 106,
|
|
"step_time": 587.4538793694228
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3708.5,
|
|
"completions/mean_length": 1262.1484375,
|
|
"completions/mean_terminated_length": 885.9746398925781,
|
|
"completions/min_length": 19.0,
|
|
"completions/min_terminated_length": 19.0,
|
|
"entropy": 3.566706972196698,
|
|
"epoch": 0.24263038548752835,
|
|
"frac_reward_zero_std": 0.84375,
|
|
"grad_norm": 0.169921875,
|
|
"kl": 0.0005286504069772491,
|
|
"learning_rate": 7.596371882086168e-07,
|
|
"loss": 0.0078,
|
|
"num_tokens": 21060904.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.41608040034770966,
|
|
"rewards/accuracy_reward_func/mean": 0.078125,
|
|
"rewards/accuracy_reward_func/std": 0.38841013610363007,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 107,
|
|
"step_time": 621.596909858752
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3543.0,
|
|
"completions/mean_length": 1123.296875,
|
|
"completions/mean_terminated_length": 787.1844787597656,
|
|
"completions/min_length": 8.5,
|
|
"completions/min_terminated_length": 8.5,
|
|
"entropy": 3.004608254879713,
|
|
"epoch": 0.24489795918367346,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.298828125,
|
|
"kl": 0.0006105130787545932,
|
|
"learning_rate": 7.573696145124716e-07,
|
|
"loss": 0.0024,
|
|
"num_tokens": 21227694.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.3587139919400215,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.3145764470100403,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 108,
|
|
"step_time": 590.9640978942625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3571.0,
|
|
"completions/mean_length": 1324.8046875,
|
|
"completions/mean_terminated_length": 837.8250732421875,
|
|
"completions/min_length": 26.0,
|
|
"completions/min_terminated_length": 26.0,
|
|
"entropy": 3.4616124592721462,
|
|
"epoch": 0.2471655328798186,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.2275390625,
|
|
"kl": 0.0006760806036254507,
|
|
"learning_rate": 7.551020408163265e-07,
|
|
"loss": -0.0509,
|
|
"num_tokens": 21423821.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.4644871801137924,
|
|
"rewards/accuracy_reward_func/mean": 0.140625,
|
|
"rewards/accuracy_reward_func/std": 0.4583333432674408,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10483578220009804,
|
|
"step": 109,
|
|
"step_time": 641.2328396155499
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3927.0,
|
|
"completions/mean_length": 1127.7734375,
|
|
"completions/mean_terminated_length": 855.1835327148438,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 3.090597476810217,
|
|
"epoch": 0.2494331065759637,
|
|
"frac_reward_zero_std": 0.8125,
|
|
"grad_norm": 0.255859375,
|
|
"kl": 0.0005724334750993876,
|
|
"learning_rate": 7.528344671201814e-07,
|
|
"loss": 0.0169,
|
|
"num_tokens": 21591224.0,
|
|
"reward": 0.11328125,
|
|
"reward_std": 0.42214976251125336,
|
|
"rewards/accuracy_reward_func/mean": 0.109375,
|
|
"rewards/accuracy_reward_func/std": 0.41878482699394226,
|
|
"rewards/format_reward_func/mean": 0.00390625,
|
|
"rewards/format_reward_func/std": 0.03125,
|
|
"step": 110,
|
|
"step_time": 599.4026822941378
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 441,
|
|
"num_input_tokens_seen": 21591224,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|