1385 lines
50 KiB
JSON
1385 lines
50 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 450,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 571.8,
|
||
|
|
"completions/max_terminated_length": 571.8,
|
||
|
|
"completions/mean_length": 210.3375,
|
||
|
|
"completions/mean_terminated_length": 210.3375,
|
||
|
|
"completions/min_length": 25.9,
|
||
|
|
"completions/min_terminated_length": 25.9,
|
||
|
|
"entropy": 0.5260405464097857,
|
||
|
|
"epoch": 0.022222222222222223,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.08984375,
|
||
|
|
"kl": 0.004743382518677208,
|
||
|
|
"learning_rate": 9.990133642141357e-07,
|
||
|
|
"loss": -0.1532900333404541,
|
||
|
|
"num_tokens": 28079.0,
|
||
|
|
"reward": 1.5,
|
||
|
|
"reward_std": 0.4325482964515686,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.525,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3935012608766556,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 10,
|
||
|
|
"step_time": 22.153983098920435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.025,
|
||
|
|
"completions/max_length": 781.7,
|
||
|
|
"completions/max_terminated_length": 520.5,
|
||
|
|
"completions/mean_length": 244.575,
|
||
|
|
"completions/mean_terminated_length": 199.82678833007813,
|
||
|
|
"completions/min_length": 11.8,
|
||
|
|
"completions/min_terminated_length": 11.8,
|
||
|
|
"entropy": 0.34257757353479973,
|
||
|
|
"epoch": 0.044444444444444446,
|
||
|
|
"frac_reward_zero_std": 0.65,
|
||
|
|
"grad_norm": 0.765625,
|
||
|
|
"kl": 0.006423782935695499,
|
||
|
|
"learning_rate": 9.956077701257707e-07,
|
||
|
|
"loss": 0.05708537697792053,
|
||
|
|
"num_tokens": 58605.0,
|
||
|
|
"reward": 1.725,
|
||
|
|
"reward_std": 0.4088952898979187,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.775,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.32741602063179015,
|
||
|
|
"rewards/format_reward_func/mean": 0.95,
|
||
|
|
"rewards/format_reward_func/std": 0.09258201122283935,
|
||
|
|
"step": 20,
|
||
|
|
"step_time": 29.545378133747725
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.025,
|
||
|
|
"completions/max_length": 941.8,
|
||
|
|
"completions/max_terminated_length": 809.2,
|
||
|
|
"completions/mean_length": 266.275,
|
||
|
|
"completions/mean_terminated_length": 228.87679138183594,
|
||
|
|
"completions/min_length": 19.2,
|
||
|
|
"completions/min_terminated_length": 19.2,
|
||
|
|
"entropy": 0.4463253241032362,
|
||
|
|
"epoch": 0.06666666666666667,
|
||
|
|
"frac_reward_zero_std": 0.5,
|
||
|
|
"grad_norm": 0.83984375,
|
||
|
|
"kl": 0.004194882212846096,
|
||
|
|
"learning_rate": 9.89787624799672e-07,
|
||
|
|
"loss": -0.03688110411167145,
|
||
|
|
"num_tokens": 90843.0,
|
||
|
|
"reward": 1.55,
|
||
|
|
"reward_std": 0.46539856791496276,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.575,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4237646222114563,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 30,
|
||
|
|
"step_time": 35.055862438119945
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 623.7,
|
||
|
|
"completions/max_terminated_length": 623.7,
|
||
|
|
"completions/mean_length": 174.3375,
|
||
|
|
"completions/mean_terminated_length": 174.3375,
|
||
|
|
"completions/min_length": 12.5,
|
||
|
|
"completions/min_terminated_length": 12.5,
|
||
|
|
"entropy": 0.4108163947239518,
|
||
|
|
"epoch": 0.08888888888888889,
|
||
|
|
"frac_reward_zero_std": 0.7,
|
||
|
|
"grad_norm": 0.86328125,
|
||
|
|
"kl": 0.004012970752546608,
|
||
|
|
"learning_rate": 9.81581283398829e-07,
|
||
|
|
"loss": -0.041548117995262146,
|
||
|
|
"num_tokens": 115550.0,
|
||
|
|
"reward": 1.45,
|
||
|
|
"reward_std": 0.3992423236370087,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.475,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.36994043588638303,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.046291005611419675,
|
||
|
|
"step": 40,
|
||
|
|
"step_time": 23.89164825808257
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0375,
|
||
|
|
"completions/max_length": 1086.7,
|
||
|
|
"completions/max_terminated_length": 770.1,
|
||
|
|
"completions/mean_length": 295.6625,
|
||
|
|
"completions/mean_terminated_length": 225.37857208251953,
|
||
|
|
"completions/min_length": 19.0,
|
||
|
|
"completions/min_terminated_length": 19.0,
|
||
|
|
"entropy": 0.3411364992149174,
|
||
|
|
"epoch": 0.1111111111111111,
|
||
|
|
"frac_reward_zero_std": 0.45,
|
||
|
|
"grad_norm": 0.1416015625,
|
||
|
|
"kl": 0.0024333317864744684,
|
||
|
|
"learning_rate": 9.710287263936483e-07,
|
||
|
|
"loss": 0.01952042579650879,
|
||
|
|
"num_tokens": 150471.0,
|
||
|
|
"reward": 1.5875,
|
||
|
|
"reward_std": 0.5131408303976059,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6375,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.43809494078159333,
|
||
|
|
"rewards/format_reward_func/mean": 0.95,
|
||
|
|
"rewards/format_reward_func/std": 0.09258201122283935,
|
||
|
|
"step": 50,
|
||
|
|
"step_time": 40.592786005605014
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 416.1,
|
||
|
|
"completions/max_terminated_length": 416.1,
|
||
|
|
"completions/mean_length": 132.075,
|
||
|
|
"completions/mean_terminated_length": 132.075,
|
||
|
|
"completions/min_length": 11.1,
|
||
|
|
"completions/min_terminated_length": 11.1,
|
||
|
|
"entropy": 0.4589052753522992,
|
||
|
|
"epoch": 0.13333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 10.5,
|
||
|
|
"kl": 0.0038458615549870957,
|
||
|
|
"learning_rate": 9.581813647811197e-07,
|
||
|
|
"loss": 0.013749442994594574,
|
||
|
|
"num_tokens": 171793.0,
|
||
|
|
"reward": 1.775,
|
||
|
|
"reward_std": 0.2839782267808914,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.8,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.24407235085964202,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 60,
|
||
|
|
"step_time": 16.592803716100754
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05,
|
||
|
|
"completions/max_length": 838.2,
|
||
|
|
"completions/max_terminated_length": 672.2,
|
||
|
|
"completions/mean_length": 310.8625,
|
||
|
|
"completions/mean_terminated_length": 223.80107421875,
|
||
|
|
"completions/min_length": 12.0,
|
||
|
|
"completions/min_terminated_length": 12.0,
|
||
|
|
"entropy": 0.39748903196305035,
|
||
|
|
"epoch": 0.15555555555555556,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 3.203125,
|
||
|
|
"kl": 0.009320074055262273,
|
||
|
|
"learning_rate": 9.431017896156073e-07,
|
||
|
|
"loss": 0.04533530175685883,
|
||
|
|
"num_tokens": 208122.0,
|
||
|
|
"reward": 1.525,
|
||
|
|
"reward_std": 0.44547489285469055,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.5875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.34551292955875396,
|
||
|
|
"rewards/format_reward_func/mean": 0.9375,
|
||
|
|
"rewards/format_reward_func/std": 0.12246559858322144,
|
||
|
|
"step": 70,
|
||
|
|
"step_time": 31.752976473141462
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.025,
|
||
|
|
"completions/max_length": 666.2,
|
||
|
|
"completions/max_terminated_length": 512.1,
|
||
|
|
"completions/mean_length": 155.05,
|
||
|
|
"completions/mean_terminated_length": 111.56607513427734,
|
||
|
|
"completions/min_length": 11.2,
|
||
|
|
"completions/min_terminated_length": 11.2,
|
||
|
|
"entropy": 0.4051839893683791,
|
||
|
|
"epoch": 0.17777777777777778,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 6.03125,
|
||
|
|
"kl": 0.006855625934451837,
|
||
|
|
"learning_rate": 9.258634670715237e-07,
|
||
|
|
"loss": 0.1526792049407959,
|
||
|
|
"num_tokens": 231274.0,
|
||
|
|
"reward": 1.625,
|
||
|
|
"reward_std": 0.36304850578308107,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.65,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.30385519564151764,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 80,
|
||
|
|
"step_time": 25.53765450278297
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0375,
|
||
|
|
"completions/max_length": 641.0,
|
||
|
|
"completions/max_terminated_length": 541.7,
|
||
|
|
"completions/mean_length": 252.0,
|
||
|
|
"completions/mean_terminated_length": 197.65595703125,
|
||
|
|
"completions/min_length": 11.9,
|
||
|
|
"completions/min_terminated_length": 11.9,
|
||
|
|
"entropy": 0.3982577932416461,
|
||
|
|
"epoch": 0.2,
|
||
|
|
"frac_reward_zero_std": 0.7,
|
||
|
|
"grad_norm": 18.5,
|
||
|
|
"kl": 0.004655605388043149,
|
||
|
|
"learning_rate": 9.065503805235137e-07,
|
||
|
|
"loss": 0.05322921872138977,
|
||
|
|
"num_tokens": 262278.0,
|
||
|
|
"reward": 1.6,
|
||
|
|
"reward_std": 0.3093269467353821,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6375,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.22768060266971588,
|
||
|
|
"rewards/format_reward_func/mean": 0.9625,
|
||
|
|
"rewards/format_reward_func/std": 0.0816463440656662,
|
||
|
|
"step": 90,
|
||
|
|
"step_time": 24.73598325951025
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 501.6,
|
||
|
|
"completions/max_terminated_length": 501.6,
|
||
|
|
"completions/mean_length": 117.4625,
|
||
|
|
"completions/mean_terminated_length": 117.4625,
|
||
|
|
"completions/min_length": 11.9,
|
||
|
|
"completions/min_terminated_length": 11.9,
|
||
|
|
"entropy": 0.4523235411616042,
|
||
|
|
"epoch": 0.2222222222222222,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.2412109375,
|
||
|
|
"kl": 0.0038861612925586543,
|
||
|
|
"learning_rate": 8.852566213878946e-07,
|
||
|
|
"loss": -0.17166446447372435,
|
||
|
|
"num_tokens": 282819.0,
|
||
|
|
"reward": 1.9,
|
||
|
|
"reward_std": 0.1796922653913498,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.9,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.17969227135181426,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 100,
|
||
|
|
"step_time": 19.58473698506132
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 364.9,
|
||
|
|
"completions/max_terminated_length": 364.9,
|
||
|
|
"completions/mean_length": 93.0125,
|
||
|
|
"completions/mean_terminated_length": 93.0125,
|
||
|
|
"completions/min_length": 10.7,
|
||
|
|
"completions/min_terminated_length": 10.7,
|
||
|
|
"entropy": 0.3852502123452723,
|
||
|
|
"epoch": 0.24444444444444444,
|
||
|
|
"frac_reward_zero_std": 0.7,
|
||
|
|
"grad_norm": 38.5,
|
||
|
|
"kl": 0.0029849309575197937,
|
||
|
|
"learning_rate": 8.620859307187338e-07,
|
||
|
|
"loss": -0.06970181465148925,
|
||
|
|
"num_tokens": 301756.0,
|
||
|
|
"reward": 1.675,
|
||
|
|
"reward_std": 0.3110164314508438,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.2929195284843445,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 110,
|
||
|
|
"step_time": 14.820625300519168
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 782.8,
|
||
|
|
"completions/max_terminated_length": 633.1,
|
||
|
|
"completions/mean_length": 221.725,
|
||
|
|
"completions/mean_terminated_length": 198.3125015258789,
|
||
|
|
"completions/min_length": 12.1,
|
||
|
|
"completions/min_terminated_length": 12.1,
|
||
|
|
"entropy": 0.40864241449162364,
|
||
|
|
"epoch": 0.26666666666666666,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 4.0625,
|
||
|
|
"kl": 0.003238640889708222,
|
||
|
|
"learning_rate": 8.371511937918617e-07,
|
||
|
|
"loss": 0.06257223486900329,
|
||
|
|
"num_tokens": 331842.0,
|
||
|
|
"reward": 1.65,
|
||
|
|
"reward_std": 0.36446867883205414,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6625,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3291133463382721,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 120,
|
||
|
|
"step_time": 29.600185937993228
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 615.2,
|
||
|
|
"completions/max_terminated_length": 424.7,
|
||
|
|
"completions/mean_length": 178.8875,
|
||
|
|
"completions/mean_terminated_length": 153.88571434020997,
|
||
|
|
"completions/min_length": 11.4,
|
||
|
|
"completions/min_terminated_length": 11.4,
|
||
|
|
"entropy": 0.36546113309450445,
|
||
|
|
"epoch": 0.28888888888888886,
|
||
|
|
"frac_reward_zero_std": 0.55,
|
||
|
|
"grad_norm": 4.03125,
|
||
|
|
"kl": 0.043059189406889686,
|
||
|
|
"learning_rate": 8.105738901391551e-07,
|
||
|
|
"loss": 0.0829508662223816,
|
||
|
|
"num_tokens": 357617.0,
|
||
|
|
"reward": 1.6625,
|
||
|
|
"reward_std": 0.3564689576625824,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3185574531555176,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 130,
|
||
|
|
"step_time": 23.732911340519784
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0375,
|
||
|
|
"completions/max_length": 794.2,
|
||
|
|
"completions/max_terminated_length": 587.0,
|
||
|
|
"completions/mean_length": 254.4625,
|
||
|
|
"completions/mean_terminated_length": 190.2952407836914,
|
||
|
|
"completions/min_length": 12.0,
|
||
|
|
"completions/min_terminated_length": 12.0,
|
||
|
|
"entropy": 0.4260031552053988,
|
||
|
|
"epoch": 0.3111111111111111,
|
||
|
|
"frac_reward_zero_std": 0.7,
|
||
|
|
"grad_norm": 2.40625,
|
||
|
|
"kl": 0.004566994193298512,
|
||
|
|
"learning_rate": 7.82483501712469e-07,
|
||
|
|
"loss": 0.1280433177947998,
|
||
|
|
"num_tokens": 389998.0,
|
||
|
|
"reward": 1.5875,
|
||
|
|
"reward_std": 0.31729506254196166,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6375,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.23484184443950654,
|
||
|
|
"rewards/format_reward_func/mean": 0.95,
|
||
|
|
"rewards/format_reward_func/std": 0.11700168251991272,
|
||
|
|
"step": 140,
|
||
|
|
"step_time": 29.948555017076433
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.025,
|
||
|
|
"completions/max_length": 481.9,
|
||
|
|
"completions/max_terminated_length": 379.7,
|
||
|
|
"completions/mean_length": 156.2375,
|
||
|
|
"completions/mean_terminated_length": 113.08333435058594,
|
||
|
|
"completions/min_length": 11.0,
|
||
|
|
"completions/min_terminated_length": 11.0,
|
||
|
|
"entropy": 0.35120144123211505,
|
||
|
|
"epoch": 0.3333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.1015625,
|
||
|
|
"kl": 0.0030827762759145115,
|
||
|
|
"learning_rate": 7.530168820605818e-07,
|
||
|
|
"loss": -0.0070336699485778805,
|
||
|
|
"num_tokens": 413489.0,
|
||
|
|
"reward": 1.5375,
|
||
|
|
"reward_std": 0.41186747550964353,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.575,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3472102552652359,
|
||
|
|
"rewards/format_reward_func/mean": 0.9625,
|
||
|
|
"rewards/format_reward_func/std": 0.0816463440656662,
|
||
|
|
"step": 150,
|
||
|
|
"step_time": 18.941299455426634
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 822.9,
|
||
|
|
"completions/max_terminated_length": 656.7,
|
||
|
|
"completions/mean_length": 179.2125,
|
||
|
|
"completions/mean_terminated_length": 154.84642868041993,
|
||
|
|
"completions/min_length": 11.8,
|
||
|
|
"completions/min_terminated_length": 11.8,
|
||
|
|
"entropy": 0.40060292216949167,
|
||
|
|
"epoch": 0.35555555555555557,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 3.8125,
|
||
|
|
"kl": 0.009431924868613351,
|
||
|
|
"learning_rate": 7.223175895924637e-07,
|
||
|
|
"loss": -0.025944623351097106,
|
||
|
|
"num_tokens": 438754.0,
|
||
|
|
"reward": 1.675,
|
||
|
|
"reward_std": 0.41999801993370056,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4010422766208649,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 160,
|
||
|
|
"step_time": 31.001335567049683
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 371.1,
|
||
|
|
"completions/max_terminated_length": 371.1,
|
||
|
|
"completions/mean_length": 124.725,
|
||
|
|
"completions/mean_terminated_length": 124.725,
|
||
|
|
"completions/min_length": 12.1,
|
||
|
|
"completions/min_terminated_length": 12.1,
|
||
|
|
"entropy": 0.39380953572690486,
|
||
|
|
"epoch": 0.37777777777777777,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.78515625,
|
||
|
|
"kl": 0.006049546646372051,
|
||
|
|
"learning_rate": 6.905351881751371e-07,
|
||
|
|
"loss": -0.037034496665000916,
|
||
|
|
"num_tokens": 458932.0,
|
||
|
|
"reward": 1.75,
|
||
|
|
"reward_std": 0.3400490075349808,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.75,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.34004901349544525,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 170,
|
||
|
|
"step_time": 14.985138600133359
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 692.3,
|
||
|
|
"completions/max_terminated_length": 692.3,
|
||
|
|
"completions/mean_length": 206.1875,
|
||
|
|
"completions/mean_terminated_length": 206.1875,
|
||
|
|
"completions/min_length": 13.7,
|
||
|
|
"completions/min_terminated_length": 13.7,
|
||
|
|
"entropy": 0.3043558836914599,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"frac_reward_zero_std": 0.5,
|
||
|
|
"grad_norm": 0.0791015625,
|
||
|
|
"kl": 0.0056259966624109484,
|
||
|
|
"learning_rate": 6.578245184735512e-07,
|
||
|
|
"loss": 0.18330377340316772,
|
||
|
|
"num_tokens": 486087.0,
|
||
|
|
"reward": 1.7,
|
||
|
|
"reward_std": 0.34984527826309203,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.7125,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3320492923259735,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 180,
|
||
|
|
"step_time": 26.418801236059515
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 484.5,
|
||
|
|
"completions/max_terminated_length": 484.5,
|
||
|
|
"completions/mean_length": 115.0125,
|
||
|
|
"completions/mean_terminated_length": 115.0125,
|
||
|
|
"completions/min_length": 11.9,
|
||
|
|
"completions/min_terminated_length": 11.9,
|
||
|
|
"entropy": 0.34211956479121,
|
||
|
|
"epoch": 0.4222222222222222,
|
||
|
|
"frac_reward_zero_std": 0.55,
|
||
|
|
"grad_norm": 95.0,
|
||
|
|
"kl": 0.006216998511573602,
|
||
|
|
"learning_rate": 6.243449435824276e-07,
|
||
|
|
"loss": -0.023849168419837953,
|
||
|
|
"num_tokens": 506532.0,
|
||
|
|
"reward": 1.5875,
|
||
|
|
"reward_std": 0.4254619300365448,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.5875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.42546194791793823,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 190,
|
||
|
|
"step_time": 19.032863819692285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 742.6,
|
||
|
|
"completions/max_terminated_length": 740.7,
|
||
|
|
"completions/mean_length": 240.3625,
|
||
|
|
"completions/mean_terminated_length": 221.21607666015626,
|
||
|
|
"completions/min_length": 10.3,
|
||
|
|
"completions/min_terminated_length": 10.3,
|
||
|
|
"entropy": 0.35800585690885783,
|
||
|
|
"epoch": 0.4444444444444444,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.1171875,
|
||
|
|
"kl": 0.004671698746193442,
|
||
|
|
"learning_rate": 5.9025957262528e-07,
|
||
|
|
"loss": 0.02241768389940262,
|
||
|
|
"num_tokens": 536945.0,
|
||
|
|
"reward": 1.7,
|
||
|
|
"reward_std": 0.37737098932266233,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.725,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3345772624015808,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 200,
|
||
|
|
"step_time": 28.164707153476776
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 554.7,
|
||
|
|
"completions/max_terminated_length": 554.7,
|
||
|
|
"completions/mean_length": 129.0125,
|
||
|
|
"completions/mean_terminated_length": 129.0125,
|
||
|
|
"completions/min_length": 10.7,
|
||
|
|
"completions/min_terminated_length": 10.7,
|
||
|
|
"entropy": 0.35905973901972177,
|
||
|
|
"epoch": 0.4666666666666667,
|
||
|
|
"frac_reward_zero_std": 0.5,
|
||
|
|
"grad_norm": 0.50390625,
|
||
|
|
"kl": 0.0016817269792369417,
|
||
|
|
"learning_rate": 5.557344661031627e-07,
|
||
|
|
"loss": -0.027326536178588868,
|
||
|
|
"num_tokens": 557854.0,
|
||
|
|
"reward": 1.6625,
|
||
|
|
"reward_std": 0.4926281929016113,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4364054501056671,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 210,
|
||
|
|
"step_time": 21.530417766235768
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 600.1,
|
||
|
|
"completions/max_terminated_length": 600.1,
|
||
|
|
"completions/mean_length": 168.9875,
|
||
|
|
"completions/mean_terminated_length": 168.9875,
|
||
|
|
"completions/min_length": 12.3,
|
||
|
|
"completions/min_terminated_length": 12.3,
|
||
|
|
"entropy": 0.4736802303697914,
|
||
|
|
"epoch": 0.4888888888888889,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.126953125,
|
||
|
|
"kl": 0.001979434840226446,
|
||
|
|
"learning_rate": 5.209378268645997e-07,
|
||
|
|
"loss": -0.03603552579879761,
|
||
|
|
"num_tokens": 582233.0,
|
||
|
|
"reward": 1.725,
|
||
|
|
"reward_std": 0.3101575791835785,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.725,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.31015759110450747,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 220,
|
||
|
|
"step_time": 23.071198928635567
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.025,
|
||
|
|
"completions/max_length": 723.2,
|
||
|
|
"completions/max_terminated_length": 645.5,
|
||
|
|
"completions/mean_length": 215.75,
|
||
|
|
"completions/mean_terminated_length": 174.425,
|
||
|
|
"completions/min_length": 12.0,
|
||
|
|
"completions/min_terminated_length": 12.0,
|
||
|
|
"entropy": 0.4890560990199447,
|
||
|
|
"epoch": 0.5111111111111111,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 5.09375,
|
||
|
|
"kl": 0.003485281078062741,
|
||
|
|
"learning_rate": 4.860391806382156e-07,
|
||
|
|
"loss": 0.11066329479217529,
|
||
|
|
"num_tokens": 610901.0,
|
||
|
|
"reward": 1.7375,
|
||
|
|
"reward_std": 0.32810779809951784,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.7625,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.2929195284843445,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.046291005611419675,
|
||
|
|
"step": 230,
|
||
|
|
"step_time": 27.49671770753339
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 521.2,
|
||
|
|
"completions/max_terminated_length": 394.2,
|
||
|
|
"completions/mean_length": 108.225,
|
||
|
|
"completions/mean_terminated_length": 85.31785736083984,
|
||
|
|
"completions/min_length": 11.1,
|
||
|
|
"completions/min_terminated_length": 11.1,
|
||
|
|
"entropy": 0.3478309240192175,
|
||
|
|
"epoch": 0.5333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.5,
|
||
|
|
"grad_norm": 0.94140625,
|
||
|
|
"kl": 0.0021312192547497943,
|
||
|
|
"learning_rate": 4.512085501204253e-07,
|
||
|
|
"loss": -0.029661089181900024,
|
||
|
|
"num_tokens": 629719.0,
|
||
|
|
"reward": 1.6375,
|
||
|
|
"reward_std": 0.41708244681358336,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6625,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.37917094230651854,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 240,
|
||
|
|
"step_time": 20.277328117284924
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 895.5,
|
||
|
|
"completions/max_terminated_length": 734.6,
|
||
|
|
"completions/mean_length": 298.4,
|
||
|
|
"completions/mean_terminated_length": 275.63392944335936,
|
||
|
|
"completions/min_length": 26.8,
|
||
|
|
"completions/min_terminated_length": 26.8,
|
||
|
|
"entropy": 0.4690596304833889,
|
||
|
|
"epoch": 0.5555555555555556,
|
||
|
|
"frac_reward_zero_std": 0.45,
|
||
|
|
"grad_norm": 12.625,
|
||
|
|
"kl": 0.0021533174171267434,
|
||
|
|
"learning_rate": 4.166156266419489e-07,
|
||
|
|
"loss": -0.16938039064407348,
|
||
|
|
"num_tokens": 664259.0,
|
||
|
|
"reward": 1.6125,
|
||
|
|
"reward_std": 0.5434043407440186,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.65,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4843781113624573,
|
||
|
|
"rewards/format_reward_func/mean": 0.9625,
|
||
|
|
"rewards/format_reward_func/std": 0.0816463440656662,
|
||
|
|
"step": 250,
|
||
|
|
"step_time": 33.734226391091944
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 741.6,
|
||
|
|
"completions/max_terminated_length": 679.4,
|
||
|
|
"completions/mean_length": 193.675,
|
||
|
|
"completions/mean_terminated_length": 173.20714416503907,
|
||
|
|
"completions/min_length": 11.8,
|
||
|
|
"completions/min_terminated_length": 11.8,
|
||
|
|
"entropy": 0.6225330086424947,
|
||
|
|
"epoch": 0.5777777777777777,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 3.890625,
|
||
|
|
"kl": 0.017426777686920046,
|
||
|
|
"learning_rate": 3.8242894344870495e-07,
|
||
|
|
"loss": 0.038351666927337644,
|
||
|
|
"num_tokens": 690625.0,
|
||
|
|
"reward": 1.5375,
|
||
|
|
"reward_std": 0.35561010539531707,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.55,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3202547788619995,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 260,
|
||
|
|
"step_time": 28.100541374552996
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 528.1,
|
||
|
|
"completions/max_terminated_length": 528.1,
|
||
|
|
"completions/mean_length": 151.1375,
|
||
|
|
"completions/mean_terminated_length": 151.1375,
|
||
|
|
"completions/min_length": 10.9,
|
||
|
|
"completions/min_terminated_length": 10.9,
|
||
|
|
"entropy": 0.41103129191324117,
|
||
|
|
"epoch": 0.6,
|
||
|
|
"frac_reward_zero_std": 0.65,
|
||
|
|
"grad_norm": 19.125,
|
||
|
|
"kl": 0.0025848565255216726,
|
||
|
|
"learning_rate": 3.488150546247778e-07,
|
||
|
|
"loss": -0.0187175914645195,
|
||
|
|
"num_tokens": 713576.0,
|
||
|
|
"reward": 1.7,
|
||
|
|
"reward_std": 0.4127540558576584,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.7125,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.384642693400383,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 270,
|
||
|
|
"step_time": 20.493615136761218
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 432.0,
|
||
|
|
"completions/max_terminated_length": 432.0,
|
||
|
|
"completions/mean_length": 123.0625,
|
||
|
|
"completions/mean_terminated_length": 123.0625,
|
||
|
|
"completions/min_length": 11.7,
|
||
|
|
"completions/min_terminated_length": 11.7,
|
||
|
|
"entropy": 0.336948611587286,
|
||
|
|
"epoch": 0.6222222222222222,
|
||
|
|
"frac_reward_zero_std": 0.5,
|
||
|
|
"grad_norm": 0.20703125,
|
||
|
|
"kl": 0.008485621147895018,
|
||
|
|
"learning_rate": 3.15937723657661e-07,
|
||
|
|
"loss": 0.037566670775413515,
|
||
|
|
"num_tokens": 734029.0,
|
||
|
|
"reward": 1.775,
|
||
|
|
"reward_std": 0.3610740154981613,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.775,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.36107403337955474,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 280,
|
||
|
|
"step_time": 17.11939077787101
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.025,
|
||
|
|
"completions/max_length": 720.1,
|
||
|
|
"completions/max_terminated_length": 317.9,
|
||
|
|
"completions/mean_length": 155.725,
|
||
|
|
"completions/mean_terminated_length": 104.97142877578736,
|
||
|
|
"completions/min_length": 11.8,
|
||
|
|
"completions/min_terminated_length": 11.8,
|
||
|
|
"entropy": 0.4226549585349858,
|
||
|
|
"epoch": 0.6444444444444445,
|
||
|
|
"frac_reward_zero_std": 0.65,
|
||
|
|
"grad_norm": 9.625,
|
||
|
|
"kl": 0.00767935578212473,
|
||
|
|
"learning_rate": 2.8395712559900874e-07,
|
||
|
|
"loss": 0.25207085609436036,
|
||
|
|
"num_tokens": 757547.0,
|
||
|
|
"reward": 1.575,
|
||
|
|
"reward_std": 0.5132734388113022,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6125,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.445256182551384,
|
||
|
|
"rewards/format_reward_func/mean": 0.9625,
|
||
|
|
"rewards/format_reward_func/std": 0.10606601536273956,
|
||
|
|
"step": 290,
|
||
|
|
"step_time": 27.44823620794341
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 551.9,
|
||
|
|
"completions/max_terminated_length": 551.9,
|
||
|
|
"completions/mean_length": 178.525,
|
||
|
|
"completions/mean_terminated_length": 178.525,
|
||
|
|
"completions/min_length": 14.1,
|
||
|
|
"completions/min_terminated_length": 14.1,
|
||
|
|
"entropy": 0.3967325270175934,
|
||
|
|
"epoch": 0.6666666666666666,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.5859375,
|
||
|
|
"kl": 0.0035352140832515034,
|
||
|
|
"learning_rate": 2.530290667078846e-07,
|
||
|
|
"loss": -0.0781548261642456,
|
||
|
|
"num_tokens": 782485.0,
|
||
|
|
"reward": 1.7375,
|
||
|
|
"reward_std": 0.3946571469306946,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.75,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3665457844734192,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 300,
|
||
|
|
"step_time": 21.438914876710623
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.025,
|
||
|
|
"completions/max_length": 940.3,
|
||
|
|
"completions/max_terminated_length": 594.3,
|
||
|
|
"completions/mean_length": 217.0875,
|
||
|
|
"completions/mean_terminated_length": 169.35892868041992,
|
||
|
|
"completions/min_length": 12.9,
|
||
|
|
"completions/min_terminated_length": 12.9,
|
||
|
|
"entropy": 0.41476303837262096,
|
||
|
|
"epoch": 0.6888888888888889,
|
||
|
|
"frac_reward_zero_std": 0.65,
|
||
|
|
"grad_norm": 0.3515625,
|
||
|
|
"kl": 0.003116685965650845,
|
||
|
|
"learning_rate": 2.23304225378328e-07,
|
||
|
|
"loss": 0.08113812804222106,
|
||
|
|
"num_tokens": 810456.0,
|
||
|
|
"reward": 1.65,
|
||
|
|
"reward_std": 0.45866142213344574,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.384642693400383,
|
||
|
|
"rewards/format_reward_func/mean": 0.9625,
|
||
|
|
"rewards/format_reward_func/std": 0.10606601536273956,
|
||
|
|
"step": 310,
|
||
|
|
"step_time": 35.4386817724444
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0375,
|
||
|
|
"completions/max_length": 1016.5,
|
||
|
|
"completions/max_terminated_length": 661.5,
|
||
|
|
"completions/mean_length": 332.925,
|
||
|
|
"completions/mean_terminated_length": 269.355362033844,
|
||
|
|
"completions/min_length": 11.1,
|
||
|
|
"completions/min_terminated_length": 11.1,
|
||
|
|
"entropy": 0.43934575584717095,
|
||
|
|
"epoch": 0.7111111111111111,
|
||
|
|
"frac_reward_zero_std": 0.45,
|
||
|
|
"grad_norm": 19.875,
|
||
|
|
"kl": 0.00665290072620337,
|
||
|
|
"learning_rate": 1.9492741804936618e-07,
|
||
|
|
"loss": 0.13840909004211427,
|
||
|
|
"num_tokens": 848386.0,
|
||
|
|
"reward": 1.575,
|
||
|
|
"reward_std": 0.5897251844406128,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6375,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4717529535293579,
|
||
|
|
"rewards/format_reward_func/mean": 0.9375,
|
||
|
|
"rewards/format_reward_func/std": 0.1767766922712326,
|
||
|
|
"step": 320,
|
||
|
|
"step_time": 38.41773249600082
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 589.8,
|
||
|
|
"completions/max_terminated_length": 415.4,
|
||
|
|
"completions/mean_length": 143.8125,
|
||
|
|
"completions/mean_terminated_length": 119.04821472167968,
|
||
|
|
"completions/min_length": 11.9,
|
||
|
|
"completions/min_terminated_length": 11.9,
|
||
|
|
"entropy": 0.5242078335955739,
|
||
|
|
"epoch": 0.7333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.55,
|
||
|
|
"grad_norm": 7.90625,
|
||
|
|
"kl": 0.006010404983805983,
|
||
|
|
"learning_rate": 1.6803689367387918e-07,
|
||
|
|
"loss": -0.01796160191297531,
|
||
|
|
"num_tokens": 871079.0,
|
||
|
|
"reward": 1.625,
|
||
|
|
"reward_std": 0.47253691256046293,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6375,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.45158677995204927,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 330,
|
||
|
|
"step_time": 22.92032507658005
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.025,
|
||
|
|
"completions/max_length": 621.5,
|
||
|
|
"completions/max_terminated_length": 326.6,
|
||
|
|
"completions/mean_length": 146.0125,
|
||
|
|
"completions/mean_terminated_length": 99.35535984039306,
|
||
|
|
"completions/min_length": 13.7,
|
||
|
|
"completions/min_terminated_length": 13.7,
|
||
|
|
"entropy": 0.4421461643651128,
|
||
|
|
"epoch": 0.7555555555555555,
|
||
|
|
"frac_reward_zero_std": 0.5,
|
||
|
|
"grad_norm": 39.75,
|
||
|
|
"kl": 0.0067155171589301345,
|
||
|
|
"learning_rate": 1.4276366018359842e-07,
|
||
|
|
"loss": 0.22347230911254884,
|
||
|
|
"num_tokens": 892896.0,
|
||
|
|
"reward": 1.75,
|
||
|
|
"reward_std": 0.3668150842189789,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.7875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.2722664475440979,
|
||
|
|
"rewards/format_reward_func/mean": 0.9625,
|
||
|
|
"rewards/format_reward_func/std": 0.10606601536273956,
|
||
|
|
"step": 340,
|
||
|
|
"step_time": 23.91549188606441
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 680.1,
|
||
|
|
"completions/max_terminated_length": 477.5,
|
||
|
|
"completions/mean_length": 148.5625,
|
||
|
|
"completions/mean_terminated_length": 123.15714292526245,
|
||
|
|
"completions/min_length": 11.6,
|
||
|
|
"completions/min_terminated_length": 11.6,
|
||
|
|
"entropy": 0.39340304215438665,
|
||
|
|
"epoch": 0.7777777777777778,
|
||
|
|
"frac_reward_zero_std": 0.7,
|
||
|
|
"grad_norm": 0.052734375,
|
||
|
|
"kl": 0.008201900802868067,
|
||
|
|
"learning_rate": 1.192308462316317e-07,
|
||
|
|
"loss": 0.015751588344573974,
|
||
|
|
"num_tokens": 916161.0,
|
||
|
|
"reward": 1.4875,
|
||
|
|
"reward_std": 0.22220884561538695,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.5,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.20411194264888763,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 350,
|
||
|
|
"step_time": 26.149945951718838
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 812.1,
|
||
|
|
"completions/max_terminated_length": 633.3,
|
||
|
|
"completions/mean_length": 248.6125,
|
||
|
|
"completions/mean_terminated_length": 224.17857208251954,
|
||
|
|
"completions/min_length": 30.9,
|
||
|
|
"completions/min_terminated_length": 30.9,
|
||
|
|
"entropy": 0.42323864391073585,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.294921875,
|
||
|
|
"kl": 0.003600113800894178,
|
||
|
|
"learning_rate": 9.755310132204297e-08,
|
||
|
|
"loss": 0.09096987247467041,
|
||
|
|
"num_tokens": 946826.0,
|
||
|
|
"reward": 1.7,
|
||
|
|
"reward_std": 0.33229822516441343,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.7125,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3084602653980255,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 360,
|
||
|
|
"step_time": 30.848398898541927
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 507.7,
|
||
|
|
"completions/max_terminated_length": 331.7,
|
||
|
|
"completions/mean_length": 167.85,
|
||
|
|
"completions/mean_terminated_length": 144.32142944335936,
|
||
|
|
"completions/min_length": 37.6,
|
||
|
|
"completions/min_terminated_length": 37.6,
|
||
|
|
"entropy": 0.5191565339453519,
|
||
|
|
"epoch": 0.8222222222222222,
|
||
|
|
"frac_reward_zero_std": 0.5,
|
||
|
|
"grad_norm": 42.25,
|
||
|
|
"kl": 0.0029767929230956726,
|
||
|
|
"learning_rate": 7.783603724899257e-08,
|
||
|
|
"loss": 0.07828059196472167,
|
||
|
|
"num_tokens": 971274.0,
|
||
|
|
"reward": 1.6,
|
||
|
|
"reward_std": 0.43631487488746645,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6125,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4082035183906555,
|
||
|
|
"rewards/format_reward_func/mean": 0.9875,
|
||
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
||
|
|
"step": 370,
|
||
|
|
"step_time": 20.00939663061872
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 210.9,
|
||
|
|
"completions/max_terminated_length": 210.9,
|
||
|
|
"completions/mean_length": 87.075,
|
||
|
|
"completions/mean_terminated_length": 87.075,
|
||
|
|
"completions/min_length": 14.3,
|
||
|
|
"completions/min_terminated_length": 14.3,
|
||
|
|
"entropy": 0.3288654647767544,
|
||
|
|
"epoch": 0.8444444444444444,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.71875,
|
||
|
|
"kl": 0.0030142821157852495,
|
||
|
|
"learning_rate": 6.017571356669182e-08,
|
||
|
|
"loss": 0.022350619733333587,
|
||
|
|
"num_tokens": 988584.0,
|
||
|
|
"reward": 1.8625,
|
||
|
|
"reward_std": 0.15149816870689392,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.8625,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.1514981746673584,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 380,
|
||
|
|
"step_time": 9.370467808935791
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 485.1,
|
||
|
|
"completions/max_terminated_length": 418.8,
|
||
|
|
"completions/mean_length": 151.8125,
|
||
|
|
"completions/mean_terminated_length": 130.91071472167968,
|
||
|
|
"completions/min_length": 10.9,
|
||
|
|
"completions/min_terminated_length": 10.9,
|
||
|
|
"entropy": 0.41746671898290516,
|
||
|
|
"epoch": 0.8666666666666667,
|
||
|
|
"frac_reward_zero_std": 0.65,
|
||
|
|
"grad_norm": 14.75,
|
||
|
|
"kl": 0.006796541407470613,
|
||
|
|
"learning_rate": 4.465816959691149e-08,
|
||
|
|
"loss": -0.13778045177459716,
|
||
|
|
"num_tokens": 1011545.0,
|
||
|
|
"reward": 1.6125,
|
||
|
|
"reward_std": 0.3564248263835907,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6375,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3147908627986908,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 390,
|
||
|
|
"step_time": 19.206426683813334
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 433.1,
|
||
|
|
"completions/max_terminated_length": 433.1,
|
||
|
|
"completions/mean_length": 130.625,
|
||
|
|
"completions/mean_terminated_length": 130.625,
|
||
|
|
"completions/min_length": 11.6,
|
||
|
|
"completions/min_terminated_length": 11.6,
|
||
|
|
"entropy": 0.46799973752349616,
|
||
|
|
"epoch": 0.8888888888888888,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.76171875,
|
||
|
|
"kl": 0.0035236999536151004,
|
||
|
|
"learning_rate": 3.135900525405427e-08,
|
||
|
|
"loss": -0.03830570280551911,
|
||
|
|
"num_tokens": 1032867.0,
|
||
|
|
"reward": 1.5875,
|
||
|
|
"reward_std": 0.345512917637825,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.5875,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.34551292955875396,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 400,
|
||
|
|
"step_time": 17.119698034226893
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 839.7,
|
||
|
|
"completions/max_terminated_length": 758.7,
|
||
|
|
"completions/mean_length": 268.6875,
|
||
|
|
"completions/mean_terminated_length": 247.98035888671876,
|
||
|
|
"completions/min_length": 21.7,
|
||
|
|
"completions/min_terminated_length": 21.7,
|
||
|
|
"entropy": 0.42415032302960753,
|
||
|
|
"epoch": 0.9111111111111111,
|
||
|
|
"frac_reward_zero_std": 0.55,
|
||
|
|
"grad_norm": 0.0274658203125,
|
||
|
|
"kl": 0.0035217325345570315,
|
||
|
|
"learning_rate": 2.0343012729971243e-08,
|
||
|
|
"loss": -0.016824692487716675,
|
||
|
|
"num_tokens": 1065190.0,
|
||
|
|
"reward": 1.5375,
|
||
|
|
"reward_std": 0.4409481525421143,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.5625,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4010422766208649,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 410,
|
||
|
|
"step_time": 31.805962641444058
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 423.3,
|
||
|
|
"completions/max_terminated_length": 423.3,
|
||
|
|
"completions/mean_length": 105.2,
|
||
|
|
"completions/mean_terminated_length": 105.2,
|
||
|
|
"completions/min_length": 13.9,
|
||
|
|
"completions/min_terminated_length": 13.9,
|
||
|
|
"entropy": 0.4501968648750335,
|
||
|
|
"epoch": 0.9333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.77734375,
|
||
|
|
"kl": 0.0039933641563425935,
|
||
|
|
"learning_rate": 1.166386083291604e-08,
|
||
|
|
"loss": -0.00834755301475525,
|
||
|
|
"num_tokens": 1085050.0,
|
||
|
|
"reward": 1.6375,
|
||
|
|
"reward_std": 0.3918039232492447,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.6375,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.39180393517017365,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 420,
|
||
|
|
"step_time": 16.97432085731998
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 580.5,
|
||
|
|
"completions/max_terminated_length": 377.9,
|
||
|
|
"completions/mean_length": 117.6625,
|
||
|
|
"completions/mean_terminated_length": 92.25357151031494,
|
||
|
|
"completions/min_length": 11.6,
|
||
|
|
"completions/min_terminated_length": 11.6,
|
||
|
|
"entropy": 0.4956971381790936,
|
||
|
|
"epoch": 0.9555555555555556,
|
||
|
|
"frac_reward_zero_std": 0.55,
|
||
|
|
"grad_norm": 0.060302734375,
|
||
|
|
"kl": 0.0021189929543652397,
|
||
|
|
"learning_rate": 5.3638335185058335e-09,
|
||
|
|
"loss": 0.16330193281173705,
|
||
|
|
"num_tokens": 1106215.0,
|
||
|
|
"reward": 1.65,
|
||
|
|
"reward_std": 0.43717372715473174,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.675,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.3808682680130005,
|
||
|
|
"rewards/format_reward_func/mean": 0.975,
|
||
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
||
|
|
"step": 430,
|
||
|
|
"step_time": 22.55132986428216
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0125,
|
||
|
|
"completions/max_length": 603.6,
|
||
|
|
"completions/max_terminated_length": 467.1,
|
||
|
|
"completions/mean_length": 169.625,
|
||
|
|
"completions/mean_terminated_length": 148.05535888671875,
|
||
|
|
"completions/min_length": 12.9,
|
||
|
|
"completions/min_terminated_length": 12.9,
|
||
|
|
"entropy": 0.4719172019511461,
|
||
|
|
"epoch": 0.9777777777777777,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 2.375,
|
||
|
|
"kl": 0.003301422532410925,
|
||
|
|
"learning_rate": 1.4736238865398765e-09,
|
||
|
|
"loss": 0.14782559871673584,
|
||
|
|
"num_tokens": 1131041.0,
|
||
|
|
"reward": 1.5125,
|
||
|
|
"reward_std": 0.5642522245645523,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.55,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4898498624563217,
|
||
|
|
"rewards/format_reward_func/mean": 0.9625,
|
||
|
|
"rewards/format_reward_func/std": 0.0816463440656662,
|
||
|
|
"step": 440,
|
||
|
|
"step_time": 23.233798024896533
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 364.2,
|
||
|
|
"completions/max_terminated_length": 364.2,
|
||
|
|
"completions/mean_length": 111.775,
|
||
|
|
"completions/mean_terminated_length": 111.775,
|
||
|
|
"completions/min_length": 12.0,
|
||
|
|
"completions/min_terminated_length": 12.0,
|
||
|
|
"entropy": 0.40269730454310776,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"frac_reward_zero_std": 0.55,
|
||
|
|
"grad_norm": 2.265625,
|
||
|
|
"kl": 0.006289730004789362,
|
||
|
|
"learning_rate": 1.2184647302626582e-11,
|
||
|
|
"loss": 0.004454012960195541,
|
||
|
|
"num_tokens": 1151755.0,
|
||
|
|
"reward": 1.65,
|
||
|
|
"reward_std": 0.43639759719371796,
|
||
|
|
"rewards/accuracy_reward_func/mean": 0.65,
|
||
|
|
"rewards/accuracy_reward_func/std": 0.4363976150751114,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"step": 450,
|
||
|
|
"step_time": 14.730636461358518
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 450,
|
||
|
|
"num_input_tokens_seen": 1151755,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|