675 lines
27 KiB
JSON
675 lines
27 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.1404494382022472,
|
|
"eval_steps": 500,
|
|
"global_step": 100,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.35,
|
|
"completions/max_length": 6342.6,
|
|
"completions/max_terminated_length": 4389.8,
|
|
"completions/mean_length": 5336.85,
|
|
"completions/mean_terminated_length": 3317.43818359375,
|
|
"completions/min_length": 3839.2,
|
|
"completions/min_terminated_length": 2220.0,
|
|
"entropy": 0.13346208892762662,
|
|
"epoch": 0.007022471910112359,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 9.64939933512099e-18,
|
|
"learning_rate": 5e-07,
|
|
"loss": -5.502499733292571e-18,
|
|
"num_tokens": 217434.0,
|
|
"reward": 0.23524999916553496,
|
|
"reward_std": 0.33813109397888186,
|
|
"rewards/compute_reward/mean": 0.23524999916553496,
|
|
"rewards/compute_reward/std": 0.33813111782073973,
|
|
"sampling/importance_sampling_ratio/max": 0.600124711072749,
|
|
"sampling/importance_sampling_ratio/mean": 0.35002241265671047,
|
|
"sampling/importance_sampling_ratio/min": 0.20000003157700258,
|
|
"sampling/sampling_logp_difference/max": 0.40465545654296875,
|
|
"sampling/sampling_logp_difference/mean": 0.012414481490850449,
|
|
"step": 5,
|
|
"step_time": 78.7855021980009
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/max_length": 8082.8,
|
|
"completions/max_terminated_length": 5149.0,
|
|
"completions/mean_length": 6318.725,
|
|
"completions/mean_terminated_length": 4152.404833984375,
|
|
"completions/min_length": 5055.0,
|
|
"completions/min_terminated_length": 3438.4,
|
|
"entropy": 0.1376650758087635,
|
|
"epoch": 0.014044943820224719,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.5075054799781356e-14,
|
|
"learning_rate": 5e-07,
|
|
"loss": -4.503626627183621e-09,
|
|
"num_tokens": 474551.0,
|
|
"reward": 0.08315625190734863,
|
|
"reward_std": 0.45229817628860475,
|
|
"rewards/compute_reward/mean": 0.08315625190734863,
|
|
"rewards/compute_reward/std": 0.4522981971502304,
|
|
"sampling/importance_sampling_ratio/max": 1.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.5000000178813935,
|
|
"sampling/importance_sampling_ratio/min": 0.2,
|
|
"sampling/sampling_logp_difference/max": 0.4511141300201416,
|
|
"sampling/sampling_logp_difference/mean": 0.01602336037904024,
|
|
"step": 10,
|
|
"step_time": 101.2872915075859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 6691.0,
|
|
"completions/max_terminated_length": 6335.8,
|
|
"completions/mean_length": 5185.35,
|
|
"completions/mean_terminated_length": 4696.5650390625,
|
|
"completions/min_length": 3638.4,
|
|
"completions/min_terminated_length": 3638.4,
|
|
"entropy": 0.1449942274019122,
|
|
"epoch": 0.021067415730337078,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2945383787155151e-07,
|
|
"learning_rate": 5e-07,
|
|
"loss": -1.8636472304933705e-06,
|
|
"num_tokens": 686789.0,
|
|
"reward": 0.3246250033378601,
|
|
"reward_std": 0.5596808314323425,
|
|
"rewards/compute_reward/mean": 0.3246250033378601,
|
|
"rewards/compute_reward/std": 0.5596808195114136,
|
|
"sampling/importance_sampling_ratio/max": 0.4000196443371508,
|
|
"sampling/importance_sampling_ratio/mean": 0.2500068536591781,
|
|
"sampling/importance_sampling_ratio/min": 4.741177590247785e-12,
|
|
"sampling/sampling_logp_difference/max": 0.5730626344680786,
|
|
"sampling/sampling_logp_difference/mean": 0.01585924383252859,
|
|
"step": 15,
|
|
"step_time": 76.25853070278535
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.525,
|
|
"completions/max_length": 6157.0,
|
|
"completions/max_terminated_length": 2626.2,
|
|
"completions/mean_length": 5558.75,
|
|
"completions/mean_terminated_length": 2014.99169921875,
|
|
"completions/min_length": 4721.0,
|
|
"completions/min_terminated_length": 1498.2,
|
|
"entropy": 0.08328314051032067,
|
|
"epoch": 0.028089887640449437,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 2.2411346435546875e-05,
|
|
"learning_rate": 5e-07,
|
|
"loss": -1.7270938769797795e-06,
|
|
"num_tokens": 913419.0,
|
|
"reward": 0.011406248807907105,
|
|
"reward_std": 0.23478415608406067,
|
|
"rewards/compute_reward/mean": 0.011406248807907105,
|
|
"rewards/compute_reward/std": 0.23478415608406067,
|
|
"sampling/importance_sampling_ratio/max": 0.6000377955271687,
|
|
"sampling/importance_sampling_ratio/mean": 0.5250090071654995,
|
|
"sampling/importance_sampling_ratio/min": 0.4000000000222174,
|
|
"sampling/sampling_logp_difference/max": 0.26677918434143066,
|
|
"sampling/sampling_logp_difference/mean": 0.0077647527679800986,
|
|
"step": 20,
|
|
"step_time": 86.48750274259365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.525,
|
|
"completions/max_length": 7358.0,
|
|
"completions/max_terminated_length": 5255.0,
|
|
"completions/mean_length": 6298.35,
|
|
"completions/mean_terminated_length": 3806.3400390625,
|
|
"completions/min_length": 4295.4,
|
|
"completions/min_terminated_length": 2676.4,
|
|
"entropy": 0.11463551484048366,
|
|
"epoch": 0.0351123595505618,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.334165275096893e-09,
|
|
"learning_rate": 5e-07,
|
|
"loss": -4.845223333660442e-10,
|
|
"num_tokens": 1168593.0,
|
|
"reward": -0.027125003933906554,
|
|
"reward_std": 0.5179957747459412,
|
|
"rewards/compute_reward/mean": -0.027125003933906554,
|
|
"rewards/compute_reward/std": 0.5179957985877991,
|
|
"sampling/importance_sampling_ratio/max": 0.8000000000302949,
|
|
"sampling/importance_sampling_ratio/mean": 0.5250000000044796,
|
|
"sampling/importance_sampling_ratio/min": 0.2,
|
|
"sampling/sampling_logp_difference/max": 0.4423918962478638,
|
|
"sampling/sampling_logp_difference/mean": 0.013001743517816067,
|
|
"step": 25,
|
|
"step_time": 100.9442321440205
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/max_length": 7867.0,
|
|
"completions/max_terminated_length": 6072.6,
|
|
"completions/mean_length": 5969.625,
|
|
"completions/mean_terminated_length": 3844.621484375,
|
|
"completions/min_length": 3876.8,
|
|
"completions/min_terminated_length": 2283.6,
|
|
"entropy": 0.1684934377670288,
|
|
"epoch": 0.042134831460674156,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": -7.479580510505457e-10,
|
|
"num_tokens": 1412506.0,
|
|
"reward": 0.21940625309944153,
|
|
"reward_std": 0.38065622448921205,
|
|
"rewards/compute_reward/mean": 0.21940625309944153,
|
|
"rewards/compute_reward/std": 0.38065624237060547,
|
|
"sampling/importance_sampling_ratio/max": 0.8000000000015144,
|
|
"sampling/importance_sampling_ratio/mean": 0.3750000059606567,
|
|
"sampling/importance_sampling_ratio/min": 0.2,
|
|
"sampling/sampling_logp_difference/max": 0.46221423149108887,
|
|
"sampling/sampling_logp_difference/mean": 0.015006156265735626,
|
|
"step": 30,
|
|
"step_time": 92.77473875220167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.225,
|
|
"completions/max_length": 5447.6,
|
|
"completions/max_terminated_length": 4491.6,
|
|
"completions/mean_length": 3979.525,
|
|
"completions/mean_terminated_length": 3340.66669921875,
|
|
"completions/min_length": 2628.0,
|
|
"completions/min_terminated_length": 2628.0,
|
|
"entropy": 0.15426228623837232,
|
|
"epoch": 0.04915730337078652,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": 5.5907651130837624e-08,
|
|
"num_tokens": 1577087.0,
|
|
"reward": 0.2894999995827675,
|
|
"reward_std": 0.4346048951148987,
|
|
"rewards/compute_reward/mean": 0.2894999995827675,
|
|
"rewards/compute_reward/std": 0.43460490703582766,
|
|
"sampling/importance_sampling_ratio/max": 0.4101849972048058,
|
|
"sampling/importance_sampling_ratio/mean": 0.2273602893020211,
|
|
"sampling/importance_sampling_ratio/min": 9.34835527009561e-05,
|
|
"sampling/sampling_logp_difference/max": 0.5941443920135498,
|
|
"sampling/sampling_logp_difference/mean": 0.014553416334092618,
|
|
"step": 35,
|
|
"step_time": 61.56992885797517
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.7,
|
|
"completions/max_length": 8080.2,
|
|
"completions/max_terminated_length": 4724.0,
|
|
"completions/mean_length": 7451.025,
|
|
"completions/mean_terminated_length": 4068.96435546875,
|
|
"completions/min_length": 6761.0,
|
|
"completions/min_terminated_length": 3524.6,
|
|
"entropy": 0.06469594687223434,
|
|
"epoch": 0.056179775280898875,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.546140993013978e-11,
|
|
"learning_rate": 5e-07,
|
|
"loss": 1.2250055136941995e-12,
|
|
"num_tokens": 1879600.0,
|
|
"reward": -0.5208124965429306,
|
|
"reward_std": 0.28857092424295844,
|
|
"rewards/compute_reward/mean": -0.5208124965429306,
|
|
"rewards/compute_reward/std": 0.2885709242895246,
|
|
"sampling/importance_sampling_ratio/max": 1.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.7,
|
|
"sampling/importance_sampling_ratio/min": 0.4,
|
|
"sampling/sampling_logp_difference/max": 0.2912771701812744,
|
|
"sampling/sampling_logp_difference/mean": 0.010922013968229293,
|
|
"step": 40,
|
|
"step_time": 122.65326870220596
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.475,
|
|
"completions/max_length": 6348.2,
|
|
"completions/max_terminated_length": 3134.0,
|
|
"completions/mean_length": 5377.575,
|
|
"completions/mean_terminated_length": 1950.0400390625,
|
|
"completions/min_length": 4405.0,
|
|
"completions/min_terminated_length": 1193.0,
|
|
"entropy": 0.12274600118398667,
|
|
"epoch": 0.06320224719101124,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": 5.91442221775651e-05,
|
|
"num_tokens": 2098759.0,
|
|
"reward": -0.26150000691413877,
|
|
"reward_std": 0.26170684695243834,
|
|
"rewards/compute_reward/mean": -0.26150000691413877,
|
|
"rewards/compute_reward/std": 0.2617068350315094,
|
|
"sampling/importance_sampling_ratio/max": 0.6020115901934332,
|
|
"sampling/importance_sampling_ratio/mean": 0.47526856281074287,
|
|
"sampling/importance_sampling_ratio/min": 0.4000000296924043,
|
|
"sampling/sampling_logp_difference/max": 0.2926159858703613,
|
|
"sampling/sampling_logp_difference/mean": 0.010471996665000916,
|
|
"step": 45,
|
|
"step_time": 85.52473701598937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 7601.6,
|
|
"completions/max_terminated_length": 6804.0,
|
|
"completions/mean_length": 5256.25,
|
|
"completions/mean_terminated_length": 4877.31513671875,
|
|
"completions/min_length": 3520.8,
|
|
"completions/min_terminated_length": 3520.8,
|
|
"entropy": 0.1676523458212614,
|
|
"epoch": 0.0702247191011236,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": -8.948970275923785e-09,
|
|
"num_tokens": 2313249.0,
|
|
"reward": 0.6231875061988831,
|
|
"reward_std": 0.30267552062869074,
|
|
"rewards/compute_reward/mean": 0.6231875061988831,
|
|
"rewards/compute_reward/std": 0.302675536647439,
|
|
"sampling/importance_sampling_ratio/max": 0.6000000545625805,
|
|
"sampling/importance_sampling_ratio/mean": 0.12500003689434863,
|
|
"sampling/importance_sampling_ratio/min": 1.509659897099789e-11,
|
|
"sampling/sampling_logp_difference/max": 0.6264592289924622,
|
|
"sampling/sampling_logp_difference/mean": 0.014229525998234748,
|
|
"step": 50,
|
|
"step_time": 76.5796675938298
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.6,
|
|
"completions/max_length": 7477.6,
|
|
"completions/max_terminated_length": 5252.6,
|
|
"completions/mean_length": 6872.8,
|
|
"completions/mean_terminated_length": 4782.11669921875,
|
|
"completions/min_length": 6005.6,
|
|
"completions/min_terminated_length": 4387.0,
|
|
"entropy": 0.12369837798178196,
|
|
"epoch": 0.07724719101123595,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": -2.9906460688816355e-19,
|
|
"num_tokens": 2592889.0,
|
|
"reward": -0.4731250017881393,
|
|
"reward_std": 0.21400290131568908,
|
|
"rewards/compute_reward/mean": -0.4731250017881393,
|
|
"rewards/compute_reward/std": 0.21400290429592134,
|
|
"sampling/importance_sampling_ratio/max": 0.8000000000400606,
|
|
"sampling/importance_sampling_ratio/mean": 0.6000000000050764,
|
|
"sampling/importance_sampling_ratio/min": 0.2,
|
|
"sampling/sampling_logp_difference/max": 0.37660536766052244,
|
|
"sampling/sampling_logp_difference/mean": 0.01498084533959627,
|
|
"step": 55,
|
|
"step_time": 109.11029338700465
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3,
|
|
"completions/max_length": 6866.0,
|
|
"completions/max_terminated_length": 6220.2,
|
|
"completions/mean_length": 5375.525,
|
|
"completions/mean_terminated_length": 4569.51435546875,
|
|
"completions/min_length": 3334.8,
|
|
"completions/min_terminated_length": 3334.8,
|
|
"entropy": 0.14546145126223564,
|
|
"epoch": 0.08426966292134831,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.715250743307479e-15,
|
|
"learning_rate": 5e-07,
|
|
"loss": -0.0002411100547760725,
|
|
"num_tokens": 2811574.0,
|
|
"reward": 0.1729999989271164,
|
|
"reward_std": 0.6171267032623291,
|
|
"rewards/compute_reward/mean": 0.1729999989271164,
|
|
"rewards/compute_reward/std": 0.6171267211437226,
|
|
"sampling/importance_sampling_ratio/max": 0.8053813435137271,
|
|
"sampling/importance_sampling_ratio/mean": 0.30101747438311577,
|
|
"sampling/importance_sampling_ratio/min": 3.9959536479727876e-08,
|
|
"sampling/sampling_logp_difference/max": 0.5940536499023438,
|
|
"sampling/sampling_logp_difference/mean": 0.015674336068332195,
|
|
"step": 60,
|
|
"step_time": 82.90924808238633
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 6714.4,
|
|
"completions/max_terminated_length": 5924.6,
|
|
"completions/mean_length": 4677.0,
|
|
"completions/mean_terminated_length": 4329.3583984375,
|
|
"completions/min_length": 3018.8,
|
|
"completions/min_terminated_length": 3018.8,
|
|
"entropy": 0.15218620076775552,
|
|
"epoch": 0.09129213483146068,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.748603820800781e-07,
|
|
"learning_rate": 5e-07,
|
|
"loss": -1.951258497001618e-08,
|
|
"num_tokens": 3004854.0,
|
|
"reward": 0.4383750021457672,
|
|
"reward_std": 0.43163261413574217,
|
|
"rewards/compute_reward/mean": 0.4383750021457672,
|
|
"rewards/compute_reward/std": 0.43163261413574217,
|
|
"sampling/importance_sampling_ratio/max": 0.6001360186763123,
|
|
"sampling/importance_sampling_ratio/mean": 0.2500173559734094,
|
|
"sampling/importance_sampling_ratio/min": 8.312385355119191e-16,
|
|
"sampling/sampling_logp_difference/max": 0.5871267795562745,
|
|
"sampling/sampling_logp_difference/mean": 0.014853741601109504,
|
|
"step": 65,
|
|
"step_time": 72.67464896581369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/max_length": 7633.4,
|
|
"completions/max_terminated_length": 5260.2,
|
|
"completions/mean_length": 6409.45,
|
|
"completions/mean_terminated_length": 4188.1984375,
|
|
"completions/min_length": 4947.8,
|
|
"completions/min_terminated_length": 3327.2,
|
|
"entropy": 0.10757671780884266,
|
|
"epoch": 0.09831460674157304,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": -2.9429700343364117e-11,
|
|
"num_tokens": 3265376.0,
|
|
"reward": 0.09690623879432678,
|
|
"reward_std": 0.5412219107151032,
|
|
"rewards/compute_reward/mean": 0.09690623879432678,
|
|
"rewards/compute_reward/std": 0.54122194647789,
|
|
"sampling/importance_sampling_ratio/max": 0.8000000000121613,
|
|
"sampling/importance_sampling_ratio/mean": 0.5000000000016139,
|
|
"sampling/importance_sampling_ratio/min": 0.2,
|
|
"sampling/sampling_logp_difference/max": 0.5057941436767578,
|
|
"sampling/sampling_logp_difference/mean": 0.01242091991007328,
|
|
"step": 70,
|
|
"step_time": 100.18304894539179
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.475,
|
|
"completions/max_length": 7485.6,
|
|
"completions/max_terminated_length": 4652.0,
|
|
"completions/mean_length": 6131.35,
|
|
"completions/mean_terminated_length": 3532.04501953125,
|
|
"completions/min_length": 4299.8,
|
|
"completions/min_terminated_length": 2692.0,
|
|
"entropy": 0.12216376923024655,
|
|
"epoch": 0.10533707865168539,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.7136335372924805e-07,
|
|
"learning_rate": 5e-07,
|
|
"loss": -1.2361371659608266e-08,
|
|
"num_tokens": 3514990.0,
|
|
"reward": 0.02084375321865082,
|
|
"reward_std": 0.6247592151165009,
|
|
"rewards/compute_reward/mean": 0.02084375321865082,
|
|
"rewards/compute_reward/std": 0.6247592031955719,
|
|
"sampling/importance_sampling_ratio/max": 0.8000003093028454,
|
|
"sampling/importance_sampling_ratio/mean": 0.475000045385957,
|
|
"sampling/importance_sampling_ratio/min": 0.20000000000001467,
|
|
"sampling/sampling_logp_difference/max": 0.46574991941452026,
|
|
"sampling/sampling_logp_difference/mean": 0.013754993863403796,
|
|
"step": 75,
|
|
"step_time": 96.75799002298155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2,
|
|
"completions/max_length": 5894.8,
|
|
"completions/max_terminated_length": 5214.2,
|
|
"completions/mean_length": 4056.425,
|
|
"completions/mean_terminated_length": 3393.43935546875,
|
|
"completions/min_length": 2331.4,
|
|
"completions/min_terminated_length": 2331.4,
|
|
"entropy": 0.14181112498044968,
|
|
"epoch": 0.11235955056179775,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.00015544891357421875,
|
|
"learning_rate": 5e-07,
|
|
"loss": 4.291571385692805e-05,
|
|
"num_tokens": 3682831.0,
|
|
"reward": 0.1677812457084656,
|
|
"reward_std": 0.41842788457870483,
|
|
"rewards/compute_reward/mean": 0.1677812457084656,
|
|
"rewards/compute_reward/std": 0.4184278786182404,
|
|
"sampling/importance_sampling_ratio/max": 0.4010849003840121,
|
|
"sampling/importance_sampling_ratio/mean": 0.20015338080880837,
|
|
"sampling/importance_sampling_ratio/min": 6.377647438378011e-10,
|
|
"sampling/sampling_logp_difference/max": 0.5928982496261597,
|
|
"sampling/sampling_logp_difference/mean": 0.01402966920286417,
|
|
"step": 80,
|
|
"step_time": 62.0829234021774
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.725,
|
|
"completions/max_length": 8033.8,
|
|
"completions/max_terminated_length": 5586.4,
|
|
"completions/mean_length": 7458.45,
|
|
"completions/mean_terminated_length": 5082.3,
|
|
"completions/min_length": 6264.2,
|
|
"completions/min_terminated_length": 4664.6,
|
|
"entropy": 0.049185825511813164,
|
|
"epoch": 0.11938202247191011,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": -1.0684439999408823e-09,
|
|
"num_tokens": 3987497.0,
|
|
"reward": -0.36359376907348634,
|
|
"reward_std": 0.35454786419868467,
|
|
"rewards/compute_reward/mean": -0.36359376907348634,
|
|
"rewards/compute_reward/std": 0.35454787611961364,
|
|
"sampling/importance_sampling_ratio/max": 1.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.725,
|
|
"sampling/importance_sampling_ratio/min": 0.2000000000015946,
|
|
"sampling/sampling_logp_difference/max": 0.4322138071060181,
|
|
"sampling/sampling_logp_difference/mean": 0.011536732874810695,
|
|
"step": 85,
|
|
"step_time": 121.9961906942015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.7,
|
|
"completions/max_length": 8083.6,
|
|
"completions/max_terminated_length": 5480.0,
|
|
"completions/mean_length": 7200.725,
|
|
"completions/mean_terminated_length": 4447.6,
|
|
"completions/min_length": 5399.0,
|
|
"completions/min_terminated_length": 3780.4,
|
|
"entropy": 0.06885569505393505,
|
|
"epoch": 0.12640449438202248,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.502887117676437e-11,
|
|
"learning_rate": 5e-07,
|
|
"loss": 3.4291680606202136e-12,
|
|
"num_tokens": 4279862.0,
|
|
"reward": -0.3385625034570694,
|
|
"reward_std": 0.6242223262786866,
|
|
"rewards/compute_reward/mean": -0.3385625034570694,
|
|
"rewards/compute_reward/std": 0.6242223501205444,
|
|
"sampling/importance_sampling_ratio/max": 1.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.7,
|
|
"sampling/importance_sampling_ratio/min": 0.20000000000054166,
|
|
"sampling/sampling_logp_difference/max": 0.36994409561157227,
|
|
"sampling/sampling_logp_difference/mean": 0.012296585179865361,
|
|
"step": 90,
|
|
"step_time": 117.92537313302746
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.4,
|
|
"completions/max_length": 6249.8,
|
|
"completions/max_terminated_length": 4446.6,
|
|
"completions/mean_length": 5328.1,
|
|
"completions/mean_terminated_length": 3106.85,
|
|
"completions/min_length": 3716.6,
|
|
"completions/min_terminated_length": 2106.2,
|
|
"entropy": 0.11026849336922169,
|
|
"epoch": 0.13342696629213482,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": -5.5089893757909183e-11,
|
|
"num_tokens": 4497202.0,
|
|
"reward": 0.19196875393390656,
|
|
"reward_std": 0.35176035165786745,
|
|
"rewards/compute_reward/mean": 0.19196875393390656,
|
|
"rewards/compute_reward/std": 0.35176035165786745,
|
|
"sampling/importance_sampling_ratio/max": 0.603415791407131,
|
|
"sampling/importance_sampling_ratio/mean": 0.4010292052433563,
|
|
"sampling/importance_sampling_ratio/min": 0.20000481952592963,
|
|
"sampling/sampling_logp_difference/max": 0.38380826711654664,
|
|
"sampling/sampling_logp_difference/mean": 0.010995614714920521,
|
|
"step": 95,
|
|
"step_time": 80.9261963216064
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.825,
|
|
"completions/max_length": 8078.4,
|
|
"completions/max_terminated_length": 2545.0,
|
|
"completions/mean_length": 7691.95,
|
|
"completions/mean_terminated_length": 2276.2333984375,
|
|
"completions/min_length": 6652.6,
|
|
"completions/min_terminated_length": 1790.2,
|
|
"entropy": 0.04812453463673592,
|
|
"epoch": 0.1404494382022472,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5e-07,
|
|
"loss": 1.299868734295816e-19,
|
|
"num_tokens": 4809424.0,
|
|
"reward": -0.5054375052452087,
|
|
"reward_std": 0.3607128977775574,
|
|
"rewards/compute_reward/mean": -0.5054375052452087,
|
|
"rewards/compute_reward/std": 0.36071292757987977,
|
|
"sampling/importance_sampling_ratio/max": 1.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.825,
|
|
"sampling/importance_sampling_ratio/min": 0.6,
|
|
"sampling/sampling_logp_difference/max": 0.1978625774383545,
|
|
"sampling/sampling_logp_difference/mean": 0.0077566314488649365,
|
|
"step": 100,
|
|
"step_time": 127.10719606040512
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 712,
|
|
"num_input_tokens_seen": 4809424,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 100,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|