13f46bb01a
Model: kikiyaa/Qwen2.5-3B-Instruct-grpo-fullfinetuning-3b-customreward Source: Original Platform
36666 lines
1.4 MiB
36666 lines
1.4 MiB
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 200,
|
|
"global_step": 10877,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0017452006980802793,
|
|
"eval_completions/max_length": 299.6928446771379,
|
|
"eval_completions/max_terminated_length": 298.3961605584642,
|
|
"eval_completions/mean_length": 209.70615183246073,
|
|
"eval_completions/mean_terminated_length": 209.30374606200746,
|
|
"eval_completions/min_length": 135.17102966841188,
|
|
"eval_completions/min_terminated_length": 135.17102966841188,
|
|
"eval_entropy": 1.0814493743745028,
|
|
"eval_frac_reward_zero_std": 0.35776614310645727,
|
|
"eval_kl": 0.0,
|
|
"eval_loss": -2.7741376470657997e-05,
|
|
"eval_num_tokens": 0.0,
|
|
"eval_reward": -0.09794938917975567,
|
|
"eval_reward_std": 0.5127268967464124,
|
|
"eval_rewards/reward_fn/mean": -0.09794938917975567,
|
|
"eval_rewards/reward_fn/std": 0.5127269140140757,
|
|
"eval_runtime": 1749.9124,
|
|
"eval_samples_per_second": 0.327,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2849910983240418,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999885321911717,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7644410355969041,
|
|
"eval_sampling/sampling_logp_difference/max": 0.3089583640947392,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021393064744315432,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 0
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 222.0,
|
|
"completions/max_terminated_length": 222.0,
|
|
"completions/mean_length": 151.625,
|
|
"completions/mean_terminated_length": 151.625,
|
|
"completions/min_length": 120.0,
|
|
"completions/min_terminated_length": 120.0,
|
|
"entropy": 1.0753526240587234,
|
|
"epoch": 9.193711501333089e-05,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"kl": 0.0,
|
|
"learning_rate": 0.0,
|
|
"loss": 0.0,
|
|
"num_tokens": 3285.0,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/reward_fn/mean": 1.0,
|
|
"rewards/reward_fn/std": 0.0,
|
|
"sampling/importance_sampling_ratio/max": 1.169453501701355,
|
|
"sampling/importance_sampling_ratio/mean": 0.9986120462417603,
|
|
"sampling/importance_sampling_ratio/min": 0.7822896838188171,
|
|
"sampling/sampling_logp_difference/max": 0.2455301284790039,
|
|
"sampling/sampling_logp_difference/mean": 0.021735867485404015,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 327.77777777777777,
|
|
"completions/max_terminated_length": 327.77777777777777,
|
|
"completions/mean_length": 217.18055555555554,
|
|
"completions/mean_terminated_length": 217.18055555555554,
|
|
"completions/min_length": 142.33333333333334,
|
|
"completions/min_terminated_length": 142.33333333333334,
|
|
"entropy": 1.0297992668218083,
|
|
"epoch": 0.0009193711501333088,
|
|
"frac_reward_zero_std": 0.5555555555555556,
|
|
"grad_norm": 0.031037449836730957,
|
|
"kl": 0.0005116063932493691,
|
|
"learning_rate": 2.757352941176471e-08,
|
|
"loss": 0.0004,
|
|
"num_tokens": 40962.0,
|
|
"reward": -0.5277777777777778,
|
|
"reward_std": 0.3398328456613753,
|
|
"rewards/reward_fn/mean": -0.5277777777777778,
|
|
"rewards/reward_fn/std": 0.3398328655295902,
|
|
"sampling/importance_sampling_ratio/max": 1.2818556096818712,
|
|
"sampling/importance_sampling_ratio/mean": 1.000329507721795,
|
|
"sampling/importance_sampling_ratio/min": 0.7555213239457872,
|
|
"sampling/sampling_logp_difference/max": 0.30765635437435573,
|
|
"sampling/sampling_logp_difference/mean": 0.020564943758977786,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.3,
|
|
"completions/max_terminated_length": 296.3,
|
|
"completions/mean_length": 200.625,
|
|
"completions/mean_terminated_length": 200.625,
|
|
"completions/min_length": 126.5,
|
|
"completions/min_terminated_length": 126.5,
|
|
"entropy": 1.0524392507970333,
|
|
"epoch": 0.0018387423002666175,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.020498275756836,
|
|
"kl": 0.0007292727361345897,
|
|
"learning_rate": 5.8210784313725494e-08,
|
|
"loss": 0.0002,
|
|
"num_tokens": 82564.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.5568657726049423,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.5568657964468002,
|
|
"sampling/importance_sampling_ratio/max": 1.291491985321045,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000877737998963,
|
|
"sampling/importance_sampling_ratio/min": 0.728471314907074,
|
|
"sampling/sampling_logp_difference/max": 0.33972215056419375,
|
|
"sampling/sampling_logp_difference/mean": 0.021487886272370815,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.3,
|
|
"completions/max_terminated_length": 270.3,
|
|
"completions/mean_length": 194.2875,
|
|
"completions/mean_terminated_length": 194.2875,
|
|
"completions/min_length": 128.7,
|
|
"completions/min_terminated_length": 128.7,
|
|
"entropy": 1.0034877896308898,
|
|
"epoch": 0.0027581134503999265,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.626897811889648,
|
|
"kl": 0.000708105590820196,
|
|
"learning_rate": 8.884803921568628e-08,
|
|
"loss": -0.0005,
|
|
"num_tokens": 123043.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6134829133749008,
|
|
"rewards/reward_fn/mean": 0.21875,
|
|
"rewards/reward_fn/std": 0.6134829312562943,
|
|
"sampling/importance_sampling_ratio/max": 1.244255793094635,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995054066181183,
|
|
"sampling/importance_sampling_ratio/min": 0.6939071506261826,
|
|
"sampling/sampling_logp_difference/max": 0.4048956513404846,
|
|
"sampling/sampling_logp_difference/mean": 0.020845489762723445,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 306.9,
|
|
"completions/max_terminated_length": 297.1,
|
|
"completions/mean_length": 211.3125,
|
|
"completions/mean_terminated_length": 208.38214416503905,
|
|
"completions/min_length": 132.6,
|
|
"completions/min_terminated_length": 132.6,
|
|
"entropy": 1.1050874687731267,
|
|
"epoch": 0.003677484600533235,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.555081367492676,
|
|
"kl": 0.0007166219835198717,
|
|
"learning_rate": 1.1948529411764705e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 168684.0,
|
|
"reward": -0.21875,
|
|
"reward_std": 0.5781980097293854,
|
|
"rewards/reward_fn/mean": -0.21875,
|
|
"rewards/reward_fn/std": 0.5781980156898499,
|
|
"sampling/importance_sampling_ratio/max": 1.284425187110901,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003512799739838,
|
|
"sampling/importance_sampling_ratio/min": 0.7563988029956817,
|
|
"sampling/sampling_logp_difference/max": 0.31729066371917725,
|
|
"sampling/sampling_logp_difference/mean": 0.0213082455098629,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.6,
|
|
"completions/max_terminated_length": 313.6,
|
|
"completions/mean_length": 205.8,
|
|
"completions/mean_terminated_length": 205.8,
|
|
"completions/min_length": 122.6,
|
|
"completions/min_terminated_length": 122.6,
|
|
"entropy": 1.0357854425907136,
|
|
"epoch": 0.004596855750666544,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.304925918579102,
|
|
"kl": 0.0006978385630645789,
|
|
"learning_rate": 1.5012254901960787e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 212108.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.5523674607276916,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.5523674786090851,
|
|
"sampling/importance_sampling_ratio/max": 1.2830299735069275,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998844981193542,
|
|
"sampling/importance_sampling_ratio/min": 0.8075290381908417,
|
|
"sampling/sampling_logp_difference/max": 0.25767743587493896,
|
|
"sampling/sampling_logp_difference/mean": 0.02069554980844259,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.4,
|
|
"completions/max_terminated_length": 331.4,
|
|
"completions/mean_length": 239.7875,
|
|
"completions/mean_terminated_length": 239.7875,
|
|
"completions/min_length": 148.3,
|
|
"completions/min_terminated_length": 148.3,
|
|
"entropy": 1.0629674822092057,
|
|
"epoch": 0.005516226900799853,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.019427070394158363,
|
|
"kl": 0.0006944268512597773,
|
|
"learning_rate": 1.8075980392156866e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 260955.0,
|
|
"reward": -0.3125,
|
|
"reward_std": 0.6532785326242447,
|
|
"rewards/reward_fn/mean": -0.3125,
|
|
"rewards/reward_fn/std": 0.6532785505056381,
|
|
"sampling/importance_sampling_ratio/max": 1.3109706163406372,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002642214298247,
|
|
"sampling/importance_sampling_ratio/min": 0.764760285615921,
|
|
"sampling/sampling_logp_difference/max": 0.3132571816444397,
|
|
"sampling/sampling_logp_difference/mean": 0.021548254787921904,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 348.8,
|
|
"completions/max_terminated_length": 344.9,
|
|
"completions/mean_length": 229.325,
|
|
"completions/mean_terminated_length": 227.15714416503906,
|
|
"completions/min_length": 138.1,
|
|
"completions/min_terminated_length": 138.1,
|
|
"entropy": 1.0357748992741107,
|
|
"epoch": 0.006435598050933162,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.994715213775635,
|
|
"kl": 0.0006824022573709953,
|
|
"learning_rate": 2.1139705882352945e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 306925.0,
|
|
"reward": -0.23125,
|
|
"reward_std": 0.5558717578649521,
|
|
"rewards/reward_fn/mean": -0.23125,
|
|
"rewards/reward_fn/std": 0.5558717578649521,
|
|
"sampling/importance_sampling_ratio/max": 1.2648253917694092,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005976200103759,
|
|
"sampling/importance_sampling_ratio/min": 0.7736561179161072,
|
|
"sampling/sampling_logp_difference/max": 0.26914299130439756,
|
|
"sampling/sampling_logp_difference/mean": 0.019995733350515365,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.2,
|
|
"completions/max_terminated_length": 310.2,
|
|
"completions/mean_length": 213.5875,
|
|
"completions/mean_terminated_length": 213.5875,
|
|
"completions/min_length": 137.6,
|
|
"completions/min_terminated_length": 137.6,
|
|
"entropy": 1.1275035381317138,
|
|
"epoch": 0.00735496920106647,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.821847915649414,
|
|
"kl": 0.0007489944571716478,
|
|
"learning_rate": 2.420343137254902e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 348940.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.5335024356842041,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.5335024356842041,
|
|
"sampling/importance_sampling_ratio/max": 1.2393616318702698,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000521004199983,
|
|
"sampling/importance_sampling_ratio/min": 0.7741144061088562,
|
|
"sampling/sampling_logp_difference/max": 0.27061805725097654,
|
|
"sampling/sampling_logp_difference/mean": 0.022378447465598583,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.9,
|
|
"completions/max_terminated_length": 305.9,
|
|
"completions/mean_length": 215.5375,
|
|
"completions/mean_terminated_length": 215.5375,
|
|
"completions/min_length": 136.7,
|
|
"completions/min_terminated_length": 136.7,
|
|
"entropy": 1.1076096400618554,
|
|
"epoch": 0.00827434035119978,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.026966840028762817,
|
|
"kl": 0.0007253596941154683,
|
|
"learning_rate": 2.7267156862745103e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 393743.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3553183853626251,
|
|
"rewards/reward_fn/mean": 0.09375,
|
|
"rewards/reward_fn/std": 0.35531840324401853,
|
|
"sampling/importance_sampling_ratio/max": 1.2905601978302002,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002325773239136,
|
|
"sampling/importance_sampling_ratio/min": 0.7468384265899658,
|
|
"sampling/sampling_logp_difference/max": 0.31531521677970886,
|
|
"sampling/sampling_logp_difference/mean": 0.022032690793275835,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.7,
|
|
"completions/max_terminated_length": 312.7,
|
|
"completions/mean_length": 215.5,
|
|
"completions/mean_terminated_length": 215.5,
|
|
"completions/min_length": 120.1,
|
|
"completions/min_terminated_length": 120.1,
|
|
"entropy": 1.0822462134063244,
|
|
"epoch": 0.009193711501333088,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.044803142547607,
|
|
"kl": 0.0007906028386059916,
|
|
"learning_rate": 3.033088235294118e-07,
|
|
"loss": 0.0005,
|
|
"num_tokens": 439087.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.5826104313135148,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.582610473036766,
|
|
"sampling/importance_sampling_ratio/max": 1.2626904606819154,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001648664474487,
|
|
"sampling/importance_sampling_ratio/min": 0.7490761339664459,
|
|
"sampling/sampling_logp_difference/max": 0.30968942642211916,
|
|
"sampling/sampling_logp_difference/mean": 0.02173212803900242,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 264.0,
|
|
"completions/max_terminated_length": 264.0,
|
|
"completions/mean_length": 183.9,
|
|
"completions/mean_terminated_length": 183.9,
|
|
"completions/min_length": 121.0,
|
|
"completions/min_terminated_length": 121.0,
|
|
"entropy": 1.075470769405365,
|
|
"epoch": 0.010113082651466397,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.02855364978313446,
|
|
"kl": 0.0007278633313035244,
|
|
"learning_rate": 3.3394607843137256e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 477567.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.5642594993114471,
|
|
"rewards/reward_fn/mean": 0.40625,
|
|
"rewards/reward_fn/std": 0.5642595171928406,
|
|
"sampling/importance_sampling_ratio/max": 1.251785933971405,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998964726924896,
|
|
"sampling/importance_sampling_ratio/min": 0.7826918303966522,
|
|
"sampling/sampling_logp_difference/max": 0.25253567695617674,
|
|
"sampling/sampling_logp_difference/mean": 0.02239386420696974,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.1,
|
|
"completions/max_terminated_length": 292.1,
|
|
"completions/mean_length": 201.75,
|
|
"completions/mean_terminated_length": 201.75,
|
|
"completions/min_length": 126.1,
|
|
"completions/min_terminated_length": 126.1,
|
|
"entropy": 1.021649543941021,
|
|
"epoch": 0.011032453801599706,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.212679862976074,
|
|
"kl": 0.0007115950233128387,
|
|
"learning_rate": 3.6458333333333337e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 519043.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.8656547248363495,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.8656547486782074,
|
|
"sampling/importance_sampling_ratio/max": 1.288342821598053,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001618027687074,
|
|
"sampling/importance_sampling_ratio/min": 0.770609176158905,
|
|
"sampling/sampling_logp_difference/max": 0.2823983430862427,
|
|
"sampling/sampling_logp_difference/mean": 0.02057406920939684,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.5,
|
|
"completions/max_terminated_length": 288.5,
|
|
"completions/mean_length": 199.3375,
|
|
"completions/mean_terminated_length": 199.3375,
|
|
"completions/min_length": 124.1,
|
|
"completions/min_terminated_length": 124.1,
|
|
"entropy": 1.0881411455571652,
|
|
"epoch": 0.011951824951733015,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.2945027351379395,
|
|
"kl": 0.0007851745387597476,
|
|
"learning_rate": 3.9522058823529414e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 562190.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.6773951411247253,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.6773951649665833,
|
|
"sampling/importance_sampling_ratio/max": 1.3473823308944701,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005816757678985,
|
|
"sampling/importance_sampling_ratio/min": 0.7659225463867188,
|
|
"sampling/sampling_logp_difference/max": 0.3466596841812134,
|
|
"sampling/sampling_logp_difference/mean": 0.022084593400359155,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.0,
|
|
"completions/max_terminated_length": 307.0,
|
|
"completions/mean_length": 210.625,
|
|
"completions/mean_terminated_length": 210.625,
|
|
"completions/min_length": 133.5,
|
|
"completions/min_terminated_length": 133.5,
|
|
"entropy": 1.0419432297348976,
|
|
"epoch": 0.012871196101866324,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.04254626855254173,
|
|
"kl": 0.0007189964184362907,
|
|
"learning_rate": 4.2585784313725495e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 608976.0,
|
|
"reward": -0.525,
|
|
"reward_std": 0.3205240786075592,
|
|
"rewards/reward_fn/mean": -0.525,
|
|
"rewards/reward_fn/std": 0.3205240905284882,
|
|
"sampling/importance_sampling_ratio/max": 1.2970109224319457,
|
|
"sampling/importance_sampling_ratio/mean": 1.000120460987091,
|
|
"sampling/importance_sampling_ratio/min": 0.7898787200450897,
|
|
"sampling/sampling_logp_difference/max": 0.2912539839744568,
|
|
"sampling/sampling_logp_difference/mean": 0.02070650216192007,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.6,
|
|
"completions/max_terminated_length": 309.6,
|
|
"completions/mean_length": 217.15,
|
|
"completions/mean_terminated_length": 217.15,
|
|
"completions/min_length": 134.9,
|
|
"completions/min_terminated_length": 134.9,
|
|
"entropy": 1.067040916532278,
|
|
"epoch": 0.013790567251999633,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.192220211029053,
|
|
"kl": 0.0007091597442922648,
|
|
"learning_rate": 4.564950980392157e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 652860.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.5939657628536225,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.5939657747745514,
|
|
"sampling/importance_sampling_ratio/max": 1.2563103079795837,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997175633907318,
|
|
"sampling/importance_sampling_ratio/min": 0.7330136477947236,
|
|
"sampling/sampling_logp_difference/max": 0.3202814102172852,
|
|
"sampling/sampling_logp_difference/mean": 0.021304642595350743,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.2,
|
|
"completions/max_terminated_length": 278.2,
|
|
"completions/mean_length": 179.15,
|
|
"completions/mean_terminated_length": 179.15,
|
|
"completions/min_length": 117.7,
|
|
"completions/min_terminated_length": 117.7,
|
|
"entropy": 1.0170505791902542,
|
|
"epoch": 0.01470993840213294,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 3.921785354614258,
|
|
"kl": 0.0007678415953705553,
|
|
"learning_rate": 4.871323529411765e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 689424.0,
|
|
"reward": -0.29375,
|
|
"reward_std": 0.3700309723615646,
|
|
"rewards/reward_fn/mean": -0.29375,
|
|
"rewards/reward_fn/std": 0.37003099620342256,
|
|
"sampling/importance_sampling_ratio/max": 1.264707350730896,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004965245723725,
|
|
"sampling/importance_sampling_ratio/min": 0.7463383138179779,
|
|
"sampling/sampling_logp_difference/max": 0.32123899459838867,
|
|
"sampling/sampling_logp_difference/mean": 0.021498026326298714,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.5,
|
|
"completions/max_terminated_length": 276.5,
|
|
"completions/mean_length": 202.5625,
|
|
"completions/mean_terminated_length": 202.5625,
|
|
"completions/min_length": 132.4,
|
|
"completions/min_terminated_length": 132.4,
|
|
"entropy": 1.0950105614960193,
|
|
"epoch": 0.01562930955226625,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.021143656224012375,
|
|
"kl": 0.0006966964305320289,
|
|
"learning_rate": 5.177696078431374e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 734909.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.3302770793437958,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.33027708530426025,
|
|
"sampling/importance_sampling_ratio/max": 1.274586832523346,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002855062484741,
|
|
"sampling/importance_sampling_ratio/min": 0.7830502808094024,
|
|
"sampling/sampling_logp_difference/max": 0.2583516001701355,
|
|
"sampling/sampling_logp_difference/mean": 0.021897630393505098,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.9,
|
|
"completions/max_terminated_length": 309.9,
|
|
"completions/mean_length": 215.975,
|
|
"completions/mean_terminated_length": 215.975,
|
|
"completions/min_length": 145.2,
|
|
"completions/min_terminated_length": 145.2,
|
|
"entropy": 1.1032920729368925,
|
|
"epoch": 0.01654868070239956,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.1822686195373535,
|
|
"kl": 0.0006948265341634397,
|
|
"learning_rate": 5.484068627450981e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 784027.0,
|
|
"reward": -0.03125,
|
|
"reward_std": 0.676500204205513,
|
|
"rewards/reward_fn/mean": -0.03125,
|
|
"rewards/reward_fn/std": 0.676500216126442,
|
|
"sampling/importance_sampling_ratio/max": 1.2685070157051086,
|
|
"sampling/importance_sampling_ratio/mean": 1.000198769569397,
|
|
"sampling/importance_sampling_ratio/min": 0.779697036743164,
|
|
"sampling/sampling_logp_difference/max": 0.26588326692581177,
|
|
"sampling/sampling_logp_difference/mean": 0.021168076153844594,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.1,
|
|
"completions/max_terminated_length": 272.1,
|
|
"completions/mean_length": 197.775,
|
|
"completions/mean_terminated_length": 197.775,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"entropy": 1.0987628474831581,
|
|
"epoch": 0.01746805185253287,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.030823295935988426,
|
|
"kl": 0.0007605922815855593,
|
|
"learning_rate": 5.790441176470589e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 826953.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.5697988569736481,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.569798880815506,
|
|
"sampling/importance_sampling_ratio/max": 1.3147042989730835,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999885618686676,
|
|
"sampling/importance_sampling_ratio/min": 0.7843068599700928,
|
|
"sampling/sampling_logp_difference/max": 0.29863872528076174,
|
|
"sampling/sampling_logp_difference/mean": 0.02155501451343298,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.0,
|
|
"completions/max_terminated_length": 314.0,
|
|
"completions/mean_length": 215.0,
|
|
"completions/mean_terminated_length": 215.0,
|
|
"completions/min_length": 129.4,
|
|
"completions/min_terminated_length": 129.4,
|
|
"entropy": 1.067587051168084,
|
|
"epoch": 0.018387423002666176,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.829469680786133,
|
|
"kl": 0.0007320545670154388,
|
|
"learning_rate": 6.096813725490197e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 870889.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.840067332983017,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.8400673389434814,
|
|
"sampling/importance_sampling_ratio/max": 1.2767957687377929,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000281572341918,
|
|
"sampling/importance_sampling_ratio/min": 0.7662912130355835,
|
|
"sampling/sampling_logp_difference/max": 0.27752859592437745,
|
|
"sampling/sampling_logp_difference/mean": 0.020865828078240157,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.018387423002666176,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0034904013961605585,
|
|
"eval_completions/max_length": 305.3438045375218,
|
|
"eval_completions/max_terminated_length": 301.54275741710296,
|
|
"eval_completions/mean_length": 209.8656195462478,
|
|
"eval_completions/mean_terminated_length": 208.9629209561706,
|
|
"eval_completions/min_length": 134.87085514834206,
|
|
"eval_completions/min_terminated_length": 134.87085514834206,
|
|
"eval_entropy": 1.085793704886711,
|
|
"eval_frac_reward_zero_std": 0.3612565445026178,
|
|
"eval_kl": 0.0007578833909915763,
|
|
"eval_loss": 4.360772800282575e-05,
|
|
"eval_num_tokens": 870889.0,
|
|
"eval_reward": -0.10056719022687609,
|
|
"eval_reward_std": 0.49869164122336823,
|
|
"eval_rewards/reward_fn/mean": -0.10056719022687609,
|
|
"eval_rewards/reward_fn/std": 0.4986916586990756,
|
|
"eval_runtime": 1772.483,
|
|
"eval_samples_per_second": 0.323,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2969242348928933,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999848889222736,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.768506151011269,
|
|
"eval_sampling/sampling_logp_difference/max": 0.30644968739770884,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021337046208311956,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 363.8,
|
|
"completions/max_terminated_length": 363.8,
|
|
"completions/mean_length": 235.2125,
|
|
"completions/mean_terminated_length": 235.2125,
|
|
"completions/min_length": 138.6,
|
|
"completions/min_terminated_length": 138.6,
|
|
"entropy": 1.0829980380833149,
|
|
"epoch": 0.019306794152799484,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.55156135559082,
|
|
"kl": 0.0007172363591962494,
|
|
"learning_rate": 6.403186274509804e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 918906.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.5179141879081726,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.5179141938686371,
|
|
"sampling/importance_sampling_ratio/max": 1.2774083375930787,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995524644851684,
|
|
"sampling/importance_sampling_ratio/min": 0.7407529830932618,
|
|
"sampling/sampling_logp_difference/max": 0.3175777614116669,
|
|
"sampling/sampling_logp_difference/mean": 0.021061371825635434,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 359.2,
|
|
"completions/max_terminated_length": 349.1,
|
|
"completions/mean_length": 236.9,
|
|
"completions/mean_terminated_length": 231.30893096923828,
|
|
"completions/min_length": 143.9,
|
|
"completions/min_terminated_length": 143.9,
|
|
"entropy": 1.1162165336310863,
|
|
"epoch": 0.020226165302932794,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.4660186767578125,
|
|
"kl": 0.000740324639264145,
|
|
"learning_rate": 6.709558823529412e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 967226.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.6329803034663201,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.6329803392291069,
|
|
"sampling/importance_sampling_ratio/max": 1.2704672336578369,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998895943164825,
|
|
"sampling/importance_sampling_ratio/min": 0.7882960200309753,
|
|
"sampling/sampling_logp_difference/max": 0.26379434466362,
|
|
"sampling/sampling_logp_difference/mean": 0.02106638755649328,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.9,
|
|
"completions/max_terminated_length": 311.9,
|
|
"completions/mean_length": 208.3875,
|
|
"completions/mean_terminated_length": 208.3875,
|
|
"completions/min_length": 131.4,
|
|
"completions/min_terminated_length": 131.4,
|
|
"entropy": 1.094875906407833,
|
|
"epoch": 0.0211455364530661,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.02850279025733471,
|
|
"kl": 0.000748710538027808,
|
|
"learning_rate": 7.01593137254902e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 1010537.0,
|
|
"reward": -0.26875,
|
|
"reward_std": 0.3905462041497231,
|
|
"rewards/reward_fn/mean": -0.26875,
|
|
"rewards/reward_fn/std": 0.390546216070652,
|
|
"sampling/importance_sampling_ratio/max": 1.2966946840286255,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003600001335144,
|
|
"sampling/importance_sampling_ratio/min": 0.7723422288894654,
|
|
"sampling/sampling_logp_difference/max": 0.2936273396015167,
|
|
"sampling/sampling_logp_difference/mean": 0.021441088058054447,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 395.5,
|
|
"completions/max_terminated_length": 354.6,
|
|
"completions/mean_length": 250.4125,
|
|
"completions/mean_terminated_length": 241.62857513427736,
|
|
"completions/min_length": 150.6,
|
|
"completions/min_terminated_length": 150.6,
|
|
"entropy": 1.126352274417877,
|
|
"epoch": 0.022064907603199412,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.024711329489946365,
|
|
"kl": 0.0007412755752739031,
|
|
"learning_rate": 7.322303921568629e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 1058626.0,
|
|
"reward": -0.30625,
|
|
"reward_std": 0.5605054497718811,
|
|
"rewards/reward_fn/mean": -0.30625,
|
|
"rewards/reward_fn/std": 0.560505473613739,
|
|
"sampling/importance_sampling_ratio/max": 1.3337203502655028,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998445868492126,
|
|
"sampling/importance_sampling_ratio/min": 0.7922480344772339,
|
|
"sampling/sampling_logp_difference/max": 0.29643134474754335,
|
|
"sampling/sampling_logp_difference/mean": 0.02184301670640707,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 318.1,
|
|
"completions/max_terminated_length": 318.1,
|
|
"completions/mean_length": 224.6125,
|
|
"completions/mean_terminated_length": 224.6125,
|
|
"completions/min_length": 149.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"entropy": 1.0780830763280391,
|
|
"epoch": 0.02298427875333272,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.025984929874539375,
|
|
"kl": 0.0007190637981693726,
|
|
"learning_rate": 7.628676470588237e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 1102371.0,
|
|
"reward": -0.58125,
|
|
"reward_std": 0.4084311366081238,
|
|
"rewards/reward_fn/mean": -0.58125,
|
|
"rewards/reward_fn/std": 0.4084311366081238,
|
|
"sampling/importance_sampling_ratio/max": 1.2953303456306458,
|
|
"sampling/importance_sampling_ratio/mean": 1.000216645002365,
|
|
"sampling/importance_sampling_ratio/min": 0.739750212430954,
|
|
"sampling/sampling_logp_difference/max": 0.31953527331352233,
|
|
"sampling/sampling_logp_difference/mean": 0.02097372431308031,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 329.6,
|
|
"completions/max_terminated_length": 307.5,
|
|
"completions/mean_length": 213.625,
|
|
"completions/mean_terminated_length": 209.9000015258789,
|
|
"completions/min_length": 129.9,
|
|
"completions/min_terminated_length": 129.9,
|
|
"entropy": 1.076523457467556,
|
|
"epoch": 0.02390364990346603,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.000105381011963,
|
|
"kl": 0.0007340486699831672,
|
|
"learning_rate": 7.935049019607844e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 1150237.0,
|
|
"reward": -0.23125,
|
|
"reward_std": 0.5087868988513946,
|
|
"rewards/reward_fn/mean": -0.23125,
|
|
"rewards/reward_fn/std": 0.5087869048118592,
|
|
"sampling/importance_sampling_ratio/max": 1.2860602378845214,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996052563190461,
|
|
"sampling/importance_sampling_ratio/min": 0.7324400305747986,
|
|
"sampling/sampling_logp_difference/max": 0.34504597783088686,
|
|
"sampling/sampling_logp_difference/mean": 0.021130956523120404,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 324.7,
|
|
"completions/max_terminated_length": 312.1,
|
|
"completions/mean_length": 232.3,
|
|
"completions/mean_terminated_length": 225.69357299804688,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"entropy": 1.0161486141383649,
|
|
"epoch": 0.024823021053599337,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 7.412708759307861,
|
|
"kl": 0.0007453218102455139,
|
|
"learning_rate": 8.241421568627452e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 1197373.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.3877462804317474,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.3877462834119797,
|
|
"sampling/importance_sampling_ratio/max": 1.3834476470947266,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002881050109864,
|
|
"sampling/importance_sampling_ratio/min": 0.7464916050434113,
|
|
"sampling/sampling_logp_difference/max": 0.40784192085266113,
|
|
"sampling/sampling_logp_difference/mean": 0.02069284338504076,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 337.3,
|
|
"completions/max_terminated_length": 337.3,
|
|
"completions/mean_length": 230.05,
|
|
"completions/mean_terminated_length": 230.05,
|
|
"completions/min_length": 142.9,
|
|
"completions/min_terminated_length": 142.9,
|
|
"entropy": 1.139074406027794,
|
|
"epoch": 0.025742392203732648,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.662224769592285,
|
|
"kl": 0.0007581617432151688,
|
|
"learning_rate": 8.54779411764706e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 1239009.0,
|
|
"reward": -0.23125,
|
|
"reward_std": 0.5634288668632508,
|
|
"rewards/reward_fn/mean": -0.23125,
|
|
"rewards/reward_fn/std": 0.5634288728237152,
|
|
"sampling/importance_sampling_ratio/max": 1.265760028362274,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004870474338532,
|
|
"sampling/importance_sampling_ratio/min": 0.7602361142635345,
|
|
"sampling/sampling_logp_difference/max": 0.29833831787109377,
|
|
"sampling/sampling_logp_difference/mean": 0.02258503921329975,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.8,
|
|
"completions/max_terminated_length": 315.8,
|
|
"completions/mean_length": 211.4,
|
|
"completions/mean_terminated_length": 211.4,
|
|
"completions/min_length": 133.2,
|
|
"completions/min_terminated_length": 133.2,
|
|
"entropy": 1.0229365639388561,
|
|
"epoch": 0.026661763353865955,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.02211562730371952,
|
|
"kl": 0.0006926543850568123,
|
|
"learning_rate": 8.854166666666668e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 1282361.0,
|
|
"reward": -0.3,
|
|
"reward_std": 0.22853160798549652,
|
|
"rewards/reward_fn/mean": -0.3,
|
|
"rewards/reward_fn/std": 0.228531613945961,
|
|
"sampling/importance_sampling_ratio/max": 1.298721969127655,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994909107685089,
|
|
"sampling/importance_sampling_ratio/min": 0.7749456644058228,
|
|
"sampling/sampling_logp_difference/max": 0.2961899220943451,
|
|
"sampling/sampling_logp_difference/mean": 0.020687919482588767,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.4,
|
|
"completions/max_terminated_length": 286.4,
|
|
"completions/mean_length": 204.0875,
|
|
"completions/mean_terminated_length": 204.0875,
|
|
"completions/min_length": 132.3,
|
|
"completions/min_terminated_length": 132.3,
|
|
"entropy": 1.0738814190030097,
|
|
"epoch": 0.027581134503999266,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.201390266418457,
|
|
"kl": 0.0007623104840604356,
|
|
"learning_rate": 9.160539215686274e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 1325656.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.5622500061988831,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.5622500300407409,
|
|
"sampling/importance_sampling_ratio/max": 1.3240941166877747,
|
|
"sampling/importance_sampling_ratio/mean": 1.000314873456955,
|
|
"sampling/importance_sampling_ratio/min": 0.7543622732162476,
|
|
"sampling/sampling_logp_difference/max": 0.32526837587356566,
|
|
"sampling/sampling_logp_difference/mean": 0.02190984096378088,
|
|
"step": 300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.2,
|
|
"completions/max_terminated_length": 294.2,
|
|
"completions/mean_length": 205.9,
|
|
"completions/mean_terminated_length": 205.9,
|
|
"completions/min_length": 126.2,
|
|
"completions/min_terminated_length": 126.2,
|
|
"entropy": 1.020055215805769,
|
|
"epoch": 0.028500505654132573,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.03616155683994293,
|
|
"kl": 0.0007322640267375391,
|
|
"learning_rate": 9.466911764705883e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 1370376.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.6827786028385162,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.6827786266803741,
|
|
"sampling/importance_sampling_ratio/max": 1.2773687243461609,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002299845218658,
|
|
"sampling/importance_sampling_ratio/min": 0.7852615237236023,
|
|
"sampling/sampling_logp_difference/max": 0.2620736837387085,
|
|
"sampling/sampling_logp_difference/mean": 0.020878583006560803,
|
|
"step": 310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.2,
|
|
"completions/max_terminated_length": 276.2,
|
|
"completions/mean_length": 198.475,
|
|
"completions/mean_terminated_length": 198.475,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 1.0327361039817333,
|
|
"epoch": 0.02941987680426588,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.836579322814941,
|
|
"kl": 0.0008560536036384292,
|
|
"learning_rate": 9.77328431372549e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 1413670.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.46159339845180514,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.46159341633319856,
|
|
"sampling/importance_sampling_ratio/max": 1.2624885797500611,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000306367874146,
|
|
"sampling/importance_sampling_ratio/min": 0.7686843037605285,
|
|
"sampling/sampling_logp_difference/max": 0.2826682835817337,
|
|
"sampling/sampling_logp_difference/mean": 0.020757396519184113,
|
|
"step": 320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 367.2,
|
|
"completions/max_terminated_length": 367.2,
|
|
"completions/mean_length": 237.575,
|
|
"completions/mean_terminated_length": 237.575,
|
|
"completions/min_length": 148.5,
|
|
"completions/min_terminated_length": 148.5,
|
|
"entropy": 1.0333620056509971,
|
|
"epoch": 0.03033924795439919,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.881727695465088,
|
|
"kl": 0.0007218352933705319,
|
|
"learning_rate": 1.00796568627451e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 1457732.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.471607381105423,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.4716073930263519,
|
|
"sampling/importance_sampling_ratio/max": 1.2482429027557373,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001922309398652,
|
|
"sampling/importance_sampling_ratio/min": 0.7963117718696594,
|
|
"sampling/sampling_logp_difference/max": 0.2448263645172119,
|
|
"sampling/sampling_logp_difference/mean": 0.020515665877610445,
|
|
"step": 330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.4,
|
|
"completions/max_terminated_length": 298.4,
|
|
"completions/mean_length": 214.2875,
|
|
"completions/mean_terminated_length": 214.2875,
|
|
"completions/min_length": 131.7,
|
|
"completions/min_terminated_length": 131.7,
|
|
"entropy": 1.0697659142315388,
|
|
"epoch": 0.0312586191045325,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.815767288208008,
|
|
"kl": 0.0007912345174190705,
|
|
"learning_rate": 1.0386029411764707e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 1499787.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.5530398905277252,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.5530398905277252,
|
|
"sampling/importance_sampling_ratio/max": 1.2939372897148131,
|
|
"sampling/importance_sampling_ratio/mean": 1.000212734937668,
|
|
"sampling/importance_sampling_ratio/min": 0.7773324012756347,
|
|
"sampling/sampling_logp_difference/max": 0.2824771046638489,
|
|
"sampling/sampling_logp_difference/mean": 0.021793242543935776,
|
|
"step": 340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.4,
|
|
"completions/max_terminated_length": 270.4,
|
|
"completions/mean_length": 191.7125,
|
|
"completions/mean_terminated_length": 191.7125,
|
|
"completions/min_length": 131.2,
|
|
"completions/min_terminated_length": 131.2,
|
|
"entropy": 1.0357216469943524,
|
|
"epoch": 0.032177990254665806,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 7.155231475830078,
|
|
"kl": 0.0008339654523297213,
|
|
"learning_rate": 1.0692401960784314e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 1537724.0,
|
|
"reward": -0.2375,
|
|
"reward_std": 0.23545143008232117,
|
|
"rewards/reward_fn/mean": -0.2375,
|
|
"rewards/reward_fn/std": 0.23545143604278565,
|
|
"sampling/importance_sampling_ratio/max": 1.315459668636322,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998289704322815,
|
|
"sampling/importance_sampling_ratio/min": 0.7348903477191925,
|
|
"sampling/sampling_logp_difference/max": 0.32952979803085325,
|
|
"sampling/sampling_logp_difference/mean": 0.021845722571015358,
|
|
"step": 350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.9,
|
|
"completions/max_terminated_length": 297.9,
|
|
"completions/mean_length": 217.0,
|
|
"completions/mean_terminated_length": 217.0,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 1.0911095909774304,
|
|
"epoch": 0.03309736140479912,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.255194664001465,
|
|
"kl": 0.0007996212814759928,
|
|
"learning_rate": 1.0998774509803921e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 1585516.0,
|
|
"reward": -0.64375,
|
|
"reward_std": 0.6160528659820557,
|
|
"rewards/reward_fn/mean": -0.64375,
|
|
"rewards/reward_fn/std": 0.6160528898239136,
|
|
"sampling/importance_sampling_ratio/max": 1.2698543906211852,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003864645957947,
|
|
"sampling/importance_sampling_ratio/min": 0.7258599668741226,
|
|
"sampling/sampling_logp_difference/max": 0.3434599757194519,
|
|
"sampling/sampling_logp_difference/mean": 0.021987689286470415,
|
|
"step": 360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.9,
|
|
"completions/max_terminated_length": 299.9,
|
|
"completions/mean_length": 215.075,
|
|
"completions/mean_terminated_length": 215.075,
|
|
"completions/min_length": 139.7,
|
|
"completions/min_terminated_length": 139.7,
|
|
"entropy": 1.0992125622928142,
|
|
"epoch": 0.03401673255493243,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.03416232019662857,
|
|
"kl": 0.0008051389224419836,
|
|
"learning_rate": 1.130514705882353e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 1632426.0,
|
|
"reward": -0.03125,
|
|
"reward_std": 0.6317772924900055,
|
|
"rewards/reward_fn/mean": -0.03125,
|
|
"rewards/reward_fn/std": 0.6317772984504699,
|
|
"sampling/importance_sampling_ratio/max": 1.27686687707901,
|
|
"sampling/importance_sampling_ratio/mean": 0.999781858921051,
|
|
"sampling/importance_sampling_ratio/min": 0.7379679620265961,
|
|
"sampling/sampling_logp_difference/max": 0.3174197942018509,
|
|
"sampling/sampling_logp_difference/mean": 0.0217969948425889,
|
|
"step": 370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.2,
|
|
"completions/max_terminated_length": 316.2,
|
|
"completions/mean_length": 219.8375,
|
|
"completions/mean_terminated_length": 219.8375,
|
|
"completions/min_length": 131.7,
|
|
"completions/min_terminated_length": 131.7,
|
|
"entropy": 1.0095836102962494,
|
|
"epoch": 0.03493610370506574,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.023060109466314316,
|
|
"kl": 0.0008119727808661992,
|
|
"learning_rate": 1.1611519607843138e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 1675877.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.4590093195438385,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.4590093374252319,
|
|
"sampling/importance_sampling_ratio/max": 1.4190353631973267,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000083804130555,
|
|
"sampling/importance_sampling_ratio/min": 0.7434727549552917,
|
|
"sampling/sampling_logp_difference/max": 0.36657058000564574,
|
|
"sampling/sampling_logp_difference/mean": 0.01990796737372875,
|
|
"step": 380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 302.7,
|
|
"completions/max_terminated_length": 278.2,
|
|
"completions/mean_length": 197.775,
|
|
"completions/mean_terminated_length": 193.8000015258789,
|
|
"completions/min_length": 128.8,
|
|
"completions/min_terminated_length": 128.8,
|
|
"entropy": 0.9564197130501271,
|
|
"epoch": 0.03585547485519904,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.02555416151881218,
|
|
"kl": 0.0007471840333892032,
|
|
"learning_rate": 1.1917892156862747e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 1718051.0,
|
|
"reward": 0.45625,
|
|
"reward_std": 0.4634471356868744,
|
|
"rewards/reward_fn/mean": 0.45625,
|
|
"rewards/reward_fn/std": 0.46344714760780337,
|
|
"sampling/importance_sampling_ratio/max": 1.3034681320190429,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996387720108032,
|
|
"sampling/importance_sampling_ratio/min": 0.7818877398967743,
|
|
"sampling/sampling_logp_difference/max": 0.305374014377594,
|
|
"sampling/sampling_logp_difference/mean": 0.020192763954401016,
|
|
"step": 390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.7,
|
|
"completions/max_terminated_length": 331.7,
|
|
"completions/mean_length": 221.2125,
|
|
"completions/mean_terminated_length": 221.2125,
|
|
"completions/min_length": 138.3,
|
|
"completions/min_terminated_length": 138.3,
|
|
"entropy": 1.0457587964832782,
|
|
"epoch": 0.03677484600533235,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.032702621072530746,
|
|
"kl": 0.000827704785115202,
|
|
"learning_rate": 1.2224264705882354e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 1764172.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3355178117752075,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.33551783561706544,
|
|
"sampling/importance_sampling_ratio/max": 1.2564478993415833,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000296652317047,
|
|
"sampling/importance_sampling_ratio/min": 0.7436484932899475,
|
|
"sampling/sampling_logp_difference/max": 0.31153822541236875,
|
|
"sampling/sampling_logp_difference/mean": 0.020990101993083952,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.03677484600533235,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0028359511343804536,
|
|
"eval_completions/max_length": 306.29842931937173,
|
|
"eval_completions/max_terminated_length": 303.9197207678883,
|
|
"eval_completions/mean_length": 211.96487783595114,
|
|
"eval_completions/mean_terminated_length": 211.27420446053017,
|
|
"eval_completions/min_length": 138.0,
|
|
"eval_completions/min_terminated_length": 138.0,
|
|
"eval_entropy": 1.082903455273227,
|
|
"eval_frac_reward_zero_std": 0.3525305410122164,
|
|
"eval_kl": 0.0007741701553322141,
|
|
"eval_loss": -1.1047100088035222e-05,
|
|
"eval_num_tokens": 1764172.0,
|
|
"eval_reward": -0.09980366492146597,
|
|
"eval_reward_std": 0.5139228775967271,
|
|
"eval_rewards/reward_fn/mean": -0.09980366492146597,
|
|
"eval_rewards/reward_fn/std": 0.5139228917437282,
|
|
"eval_runtime": 1781.0892,
|
|
"eval_samples_per_second": 0.322,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.275079117395491,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000452339961268,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7690545400347801,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2914872598845713,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021694860621038533,
|
|
"eval_steps_per_second": 0.04,
|
|
"step": 400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.9,
|
|
"completions/max_terminated_length": 270.9,
|
|
"completions/mean_length": 191.9,
|
|
"completions/mean_terminated_length": 191.9,
|
|
"completions/min_length": 128.7,
|
|
"completions/min_terminated_length": 128.7,
|
|
"entropy": 0.9748659297823906,
|
|
"epoch": 0.03769421715546566,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.02381434105336666,
|
|
"kl": 0.0007814693031832576,
|
|
"learning_rate": 1.2530637254901961e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 1804452.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.43009520769119264,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.4300952196121216,
|
|
"sampling/importance_sampling_ratio/max": 1.3290038228034973,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001201629638672,
|
|
"sampling/importance_sampling_ratio/min": 0.7543022692203522,
|
|
"sampling/sampling_logp_difference/max": 0.3361420124769211,
|
|
"sampling/sampling_logp_difference/mean": 0.02122024167329073,
|
|
"step": 410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 252.0,
|
|
"completions/max_terminated_length": 252.0,
|
|
"completions/mean_length": 187.875,
|
|
"completions/mean_terminated_length": 187.875,
|
|
"completions/min_length": 128.3,
|
|
"completions/min_terminated_length": 128.3,
|
|
"entropy": 1.0954497002065182,
|
|
"epoch": 0.03861358830559897,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.232787609100342,
|
|
"kl": 0.0008645617057482013,
|
|
"learning_rate": 1.2837009803921568e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 1845370.0,
|
|
"reward": -0.31875,
|
|
"reward_std": 0.4481493324041367,
|
|
"rewards/reward_fn/mean": -0.31875,
|
|
"rewards/reward_fn/std": 0.44814935624599456,
|
|
"sampling/importance_sampling_ratio/max": 1.2872082710266113,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000244319438933,
|
|
"sampling/importance_sampling_ratio/min": 0.6757602855563164,
|
|
"sampling/sampling_logp_difference/max": 0.566975736618042,
|
|
"sampling/sampling_logp_difference/mean": 0.022479208186268806,
|
|
"step": 420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 259.3,
|
|
"completions/max_terminated_length": 259.3,
|
|
"completions/mean_length": 195.05,
|
|
"completions/mean_terminated_length": 195.05,
|
|
"completions/min_length": 141.6,
|
|
"completions/min_terminated_length": 141.6,
|
|
"entropy": 1.0407501764595508,
|
|
"epoch": 0.03953295945573228,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.02399292029440403,
|
|
"kl": 0.0007973959025548538,
|
|
"learning_rate": 1.3143382352941177e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 1884966.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.6941522836685181,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.694152319431305,
|
|
"sampling/importance_sampling_ratio/max": 1.2793169736862182,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003010928630829,
|
|
"sampling/importance_sampling_ratio/min": 0.7122344195842742,
|
|
"sampling/sampling_logp_difference/max": 0.3782411336898804,
|
|
"sampling/sampling_logp_difference/mean": 0.02117371093481779,
|
|
"step": 430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 257.2,
|
|
"completions/max_terminated_length": 257.2,
|
|
"completions/mean_length": 189.025,
|
|
"completions/mean_terminated_length": 189.025,
|
|
"completions/min_length": 122.2,
|
|
"completions/min_terminated_length": 122.2,
|
|
"entropy": 0.9857624679803848,
|
|
"epoch": 0.04045233060586559,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 8.941465377807617,
|
|
"kl": 0.0007763271081785206,
|
|
"learning_rate": 1.3449754901960785e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 1921568.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.3700077474117279,
|
|
"rewards/reward_fn/mean": 0.34375,
|
|
"rewards/reward_fn/std": 0.3700077533721924,
|
|
"sampling/importance_sampling_ratio/max": 1.279254174232483,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998914003372192,
|
|
"sampling/importance_sampling_ratio/min": 0.7792259633541108,
|
|
"sampling/sampling_logp_difference/max": 0.28018001317977903,
|
|
"sampling/sampling_logp_difference/mean": 0.020629195868968962,
|
|
"step": 440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 313.2,
|
|
"completions/max_terminated_length": 309.1,
|
|
"completions/mean_length": 216.2875,
|
|
"completions/mean_terminated_length": 208.6375,
|
|
"completions/min_length": 136.5,
|
|
"completions/min_terminated_length": 136.5,
|
|
"entropy": 0.9929107904434205,
|
|
"epoch": 0.0413717017559989,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.332141876220703,
|
|
"kl": 0.0007418615419737762,
|
|
"learning_rate": 1.3756127450980394e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 1966583.0,
|
|
"reward": -0.39375,
|
|
"reward_std": 0.6334246933460236,
|
|
"rewards/reward_fn/mean": -0.39375,
|
|
"rewards/reward_fn/std": 0.633424723148346,
|
|
"sampling/importance_sampling_ratio/max": 1.3046724438667296,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997773408889771,
|
|
"sampling/importance_sampling_ratio/min": 0.7649337887763977,
|
|
"sampling/sampling_logp_difference/max": 0.3193891406059265,
|
|
"sampling/sampling_logp_difference/mean": 0.020112727768719196,
|
|
"step": 450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.2,
|
|
"completions/max_terminated_length": 275.2,
|
|
"completions/mean_length": 187.3375,
|
|
"completions/mean_terminated_length": 187.3375,
|
|
"completions/min_length": 123.9,
|
|
"completions/min_terminated_length": 123.9,
|
|
"entropy": 1.027133221924305,
|
|
"epoch": 0.0422910729061322,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.107820987701416,
|
|
"kl": 0.0008454843980871373,
|
|
"learning_rate": 1.40625e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 2006882.0,
|
|
"reward": -0.45625,
|
|
"reward_std": 0.5439153432846069,
|
|
"rewards/reward_fn/mean": -0.45625,
|
|
"rewards/reward_fn/std": 0.5439153611660004,
|
|
"sampling/importance_sampling_ratio/max": 1.3032878637313843,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005386471748352,
|
|
"sampling/importance_sampling_ratio/min": 0.7848802864551544,
|
|
"sampling/sampling_logp_difference/max": 0.29207401871681216,
|
|
"sampling/sampling_logp_difference/mean": 0.021029272116720676,
|
|
"step": 460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 310.1,
|
|
"completions/max_terminated_length": 305.3,
|
|
"completions/mean_length": 231.05,
|
|
"completions/mean_terminated_length": 229.3732147216797,
|
|
"completions/min_length": 158.4,
|
|
"completions/min_terminated_length": 158.4,
|
|
"entropy": 1.1479732871055603,
|
|
"epoch": 0.043210444056265514,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.885332107543945,
|
|
"kl": 0.0008643940520414617,
|
|
"learning_rate": 1.4368872549019608e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 2053974.0,
|
|
"reward": -0.26875,
|
|
"reward_std": 0.6387705266475677,
|
|
"rewards/reward_fn/mean": -0.26875,
|
|
"rewards/reward_fn/std": 0.6387705385684967,
|
|
"sampling/importance_sampling_ratio/max": 1.266026759147644,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996416211128235,
|
|
"sampling/importance_sampling_ratio/min": 0.7661808729171753,
|
|
"sampling/sampling_logp_difference/max": 0.2700227200984955,
|
|
"sampling/sampling_logp_difference/mean": 0.02255709283053875,
|
|
"step": 470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 340.8,
|
|
"completions/max_terminated_length": 340.8,
|
|
"completions/mean_length": 233.8,
|
|
"completions/mean_terminated_length": 233.8,
|
|
"completions/min_length": 151.9,
|
|
"completions/min_terminated_length": 151.9,
|
|
"entropy": 1.0974321976304053,
|
|
"epoch": 0.044129815206398824,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.995086193084717,
|
|
"kl": 0.0011237188922677888,
|
|
"learning_rate": 1.4675245098039217e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 2101638.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.5576324224472046,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.557632428407669,
|
|
"sampling/importance_sampling_ratio/max": 1.265100634098053,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995315313339234,
|
|
"sampling/importance_sampling_ratio/min": 0.731876790523529,
|
|
"sampling/sampling_logp_difference/max": 0.34230992794036863,
|
|
"sampling/sampling_logp_difference/mean": 0.022754467092454433,
|
|
"step": 480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 332.2,
|
|
"completions/max_terminated_length": 308.2,
|
|
"completions/mean_length": 227.625,
|
|
"completions/mean_terminated_length": 223.8107147216797,
|
|
"completions/min_length": 151.4,
|
|
"completions/min_terminated_length": 151.4,
|
|
"entropy": 1.1766859948635102,
|
|
"epoch": 0.045049186356532135,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.976254940032959,
|
|
"kl": 0.001084200566401705,
|
|
"learning_rate": 1.4981617647058824e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 2146832.0,
|
|
"reward": -0.16875,
|
|
"reward_std": 0.8188398405909538,
|
|
"rewards/reward_fn/mean": -0.16875,
|
|
"rewards/reward_fn/std": 0.8188398584723473,
|
|
"sampling/importance_sampling_ratio/max": 1.2931214451789856,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000166296958923,
|
|
"sampling/importance_sampling_ratio/min": 0.7744659781455994,
|
|
"sampling/sampling_logp_difference/max": 0.27478190660476687,
|
|
"sampling/sampling_logp_difference/mean": 0.02285472545772791,
|
|
"step": 490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.1,
|
|
"completions/max_terminated_length": 310.1,
|
|
"completions/mean_length": 225.25,
|
|
"completions/mean_terminated_length": 225.25,
|
|
"completions/min_length": 148.2,
|
|
"completions/min_terminated_length": 148.2,
|
|
"entropy": 1.027709399908781,
|
|
"epoch": 0.04596855750666544,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.856915473937988,
|
|
"kl": 0.0011877536860993132,
|
|
"learning_rate": 1.5287990196078434e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 2192860.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.6033036708831787,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.6033036708831787,
|
|
"sampling/importance_sampling_ratio/max": 1.2909950613975525,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999686121940613,
|
|
"sampling/importance_sampling_ratio/min": 0.7839114427566528,
|
|
"sampling/sampling_logp_difference/max": 0.2868601202964783,
|
|
"sampling/sampling_logp_difference/mean": 0.02084048669785261,
|
|
"step": 500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.0,
|
|
"completions/max_terminated_length": 302.0,
|
|
"completions/mean_length": 207.175,
|
|
"completions/mean_terminated_length": 207.175,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 1.0776195771992207,
|
|
"epoch": 0.04688792865679875,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.848415374755859,
|
|
"kl": 0.0015265618094417732,
|
|
"learning_rate": 1.559436274509804e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 2235474.0,
|
|
"reward": -0.2,
|
|
"reward_std": 0.33271285593509675,
|
|
"rewards/reward_fn/mean": -0.2,
|
|
"rewards/reward_fn/std": 0.33271287977695463,
|
|
"sampling/importance_sampling_ratio/max": 1.3018338799476623,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002345502376557,
|
|
"sampling/importance_sampling_ratio/min": 0.7495401531457901,
|
|
"sampling/sampling_logp_difference/max": 0.32740806937217715,
|
|
"sampling/sampling_logp_difference/mean": 0.02211202085018158,
|
|
"step": 510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 327.4,
|
|
"completions/max_terminated_length": 327.4,
|
|
"completions/mean_length": 199.1625,
|
|
"completions/mean_terminated_length": 199.1625,
|
|
"completions/min_length": 118.2,
|
|
"completions/min_terminated_length": 118.2,
|
|
"entropy": 1.0149781927466393,
|
|
"epoch": 0.04780729980693206,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.296606063842773,
|
|
"kl": 0.0015358019700215664,
|
|
"learning_rate": 1.5900735294117648e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 2276727.0,
|
|
"reward": -0.43125,
|
|
"reward_std": 0.6722564369440078,
|
|
"rewards/reward_fn/mean": -0.43125,
|
|
"rewards/reward_fn/std": 0.6722564429044724,
|
|
"sampling/importance_sampling_ratio/max": 1.3543675780296325,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003010928630829,
|
|
"sampling/importance_sampling_ratio/min": 0.7615984201431274,
|
|
"sampling/sampling_logp_difference/max": 0.3310415744781494,
|
|
"sampling/sampling_logp_difference/mean": 0.021809526719152927,
|
|
"step": 520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.6,
|
|
"completions/max_terminated_length": 308.6,
|
|
"completions/mean_length": 214.6875,
|
|
"completions/mean_terminated_length": 214.6875,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 1.057837551087141,
|
|
"epoch": 0.048726670957065364,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.374033451080322,
|
|
"kl": 0.0015048259701870847,
|
|
"learning_rate": 1.6207107843137257e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 2318302.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.7126178503036499,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.7126178562641143,
|
|
"sampling/importance_sampling_ratio/max": 1.238474726676941,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000241696834564,
|
|
"sampling/importance_sampling_ratio/min": 0.7692290544509888,
|
|
"sampling/sampling_logp_difference/max": 0.2738600611686707,
|
|
"sampling/sampling_logp_difference/mean": 0.021399349719285966,
|
|
"step": 530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.5,
|
|
"completions/max_terminated_length": 305.5,
|
|
"completions/mean_length": 212.95,
|
|
"completions/mean_terminated_length": 212.95,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 1.0474265605211257,
|
|
"epoch": 0.049646042107198675,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.015708923339844,
|
|
"kl": 0.0019162637305271345,
|
|
"learning_rate": 1.6513480392156864e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 2361114.0,
|
|
"reward": -0.21875,
|
|
"reward_std": 0.6456482112407684,
|
|
"rewards/reward_fn/mean": -0.21875,
|
|
"rewards/reward_fn/std": 0.6456482529640197,
|
|
"sampling/importance_sampling_ratio/max": 1.2911459445953368,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999278366565705,
|
|
"sampling/importance_sampling_ratio/min": 0.777782016992569,
|
|
"sampling/sampling_logp_difference/max": 0.287069833278656,
|
|
"sampling/sampling_logp_difference/mean": 0.021811645198613405,
|
|
"step": 540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.6,
|
|
"completions/max_terminated_length": 300.6,
|
|
"completions/mean_length": 207.7,
|
|
"completions/mean_terminated_length": 207.7,
|
|
"completions/min_length": 124.5,
|
|
"completions/min_terminated_length": 124.5,
|
|
"entropy": 0.9286081437021494,
|
|
"epoch": 0.050565413257331986,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.826114654541016,
|
|
"kl": 0.001771631084557157,
|
|
"learning_rate": 1.6819852941176473e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 2401458.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.6216493517160415,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.6216493636369705,
|
|
"sampling/importance_sampling_ratio/max": 1.300283133983612,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003273010253906,
|
|
"sampling/importance_sampling_ratio/min": 0.7246978104114532,
|
|
"sampling/sampling_logp_difference/max": 0.356741726398468,
|
|
"sampling/sampling_logp_difference/mean": 0.020618940703570843,
|
|
"step": 550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.1,
|
|
"completions/max_terminated_length": 276.1,
|
|
"completions/mean_length": 186.6,
|
|
"completions/mean_terminated_length": 186.6,
|
|
"completions/min_length": 127.6,
|
|
"completions/min_terminated_length": 127.6,
|
|
"entropy": 1.0070199850946664,
|
|
"epoch": 0.051484784407465296,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.0504374504089355,
|
|
"kl": 0.0017657156095083337,
|
|
"learning_rate": 1.712622549019608e-06,
|
|
"loss": -0.0005,
|
|
"num_tokens": 2443154.0,
|
|
"reward": -0.2625,
|
|
"reward_std": 0.6940361440181733,
|
|
"rewards/reward_fn/mean": -0.2625,
|
|
"rewards/reward_fn/std": 0.6940361678600311,
|
|
"sampling/importance_sampling_ratio/max": 1.2626020669937135,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996123135089874,
|
|
"sampling/importance_sampling_ratio/min": 0.7820769190788269,
|
|
"sampling/sampling_logp_difference/max": 0.26067387461662295,
|
|
"sampling/sampling_logp_difference/mean": 0.021085712127387524,
|
|
"step": 560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 353.7,
|
|
"completions/max_terminated_length": 353.7,
|
|
"completions/mean_length": 218.15,
|
|
"completions/mean_terminated_length": 218.15,
|
|
"completions/min_length": 114.2,
|
|
"completions/min_terminated_length": 114.2,
|
|
"entropy": 1.0417570002377032,
|
|
"epoch": 0.0524041555575986,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.932330131530762,
|
|
"kl": 0.002104779648652766,
|
|
"learning_rate": 1.7432598039215688e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 2489278.0,
|
|
"reward": -0.30625,
|
|
"reward_std": 0.5829227596521378,
|
|
"rewards/reward_fn/mean": -0.30625,
|
|
"rewards/reward_fn/std": 0.5829227954149246,
|
|
"sampling/importance_sampling_ratio/max": 1.2435389757156372,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000598549842834,
|
|
"sampling/importance_sampling_ratio/min": 0.7541009902954101,
|
|
"sampling/sampling_logp_difference/max": 0.29438533782958987,
|
|
"sampling/sampling_logp_difference/mean": 0.0215120536275208,
|
|
"step": 570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 339.0,
|
|
"completions/max_terminated_length": 326.0,
|
|
"completions/mean_length": 228.5625,
|
|
"completions/mean_terminated_length": 217.375,
|
|
"completions/min_length": 145.7,
|
|
"completions/min_terminated_length": 145.7,
|
|
"entropy": 1.0102458983659743,
|
|
"epoch": 0.05332352670773191,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.449970722198486,
|
|
"kl": 0.0017665868123003746,
|
|
"learning_rate": 1.7738970588235297e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 2534923.0,
|
|
"reward": -0.21875,
|
|
"reward_std": 0.8699533045291901,
|
|
"rewards/reward_fn/mean": -0.21875,
|
|
"rewards/reward_fn/std": 0.8699533402919769,
|
|
"sampling/importance_sampling_ratio/max": 1.253454339504242,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003077864646912,
|
|
"sampling/importance_sampling_ratio/min": 0.7230043232440948,
|
|
"sampling/sampling_logp_difference/max": 0.34247471690177916,
|
|
"sampling/sampling_logp_difference/mean": 0.020967343542724847,
|
|
"step": 580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.7,
|
|
"completions/max_terminated_length": 289.7,
|
|
"completions/mean_length": 190.15,
|
|
"completions/mean_terminated_length": 190.15,
|
|
"completions/min_length": 121.0,
|
|
"completions/min_terminated_length": 121.0,
|
|
"entropy": 1.0787615150213241,
|
|
"epoch": 0.05424289785786522,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.062219273298978806,
|
|
"kl": 0.0014621297348639927,
|
|
"learning_rate": 1.8045343137254904e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 2575151.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.2748226135969162,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.2748226195573807,
|
|
"sampling/importance_sampling_ratio/max": 1.2410045862197876,
|
|
"sampling/importance_sampling_ratio/mean": 1.000088530778885,
|
|
"sampling/importance_sampling_ratio/min": 0.7797856152057647,
|
|
"sampling/sampling_logp_difference/max": 0.25548909306526185,
|
|
"sampling/sampling_logp_difference/mean": 0.022826541773974897,
|
|
"step": 590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.7,
|
|
"completions/max_terminated_length": 299.7,
|
|
"completions/mean_length": 217.8125,
|
|
"completions/mean_terminated_length": 217.8125,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 1.0992080330848695,
|
|
"epoch": 0.05516226900799853,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.946908950805664,
|
|
"kl": 0.0015387784260383341,
|
|
"learning_rate": 1.8351715686274509e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 2622464.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.49239068627357485,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.4923907041549683,
|
|
"sampling/importance_sampling_ratio/max": 1.3550460577011108,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002450406551362,
|
|
"sampling/importance_sampling_ratio/min": 0.7415886342525482,
|
|
"sampling/sampling_logp_difference/max": 0.3970524609088898,
|
|
"sampling/sampling_logp_difference/mean": 0.022138751484453677,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.05516226900799853,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 291.7870855148342,
|
|
"eval_completions/max_terminated_length": 290.39092495637,
|
|
"eval_completions/mean_length": 203.46684118673647,
|
|
"eval_completions/mean_terminated_length": 203.13759049820027,
|
|
"eval_completions/min_length": 133.29493891797557,
|
|
"eval_completions/min_terminated_length": 133.29493891797557,
|
|
"eval_entropy": 1.0725050150411914,
|
|
"eval_frac_reward_zero_std": 0.3717277486910995,
|
|
"eval_kl": 0.0013345682309464232,
|
|
"eval_loss": 9.026547922985628e-05,
|
|
"eval_num_tokens": 2622464.0,
|
|
"eval_reward": -0.07362565445026178,
|
|
"eval_reward_std": 0.5034478372564283,
|
|
"eval_rewards/reward_fn/mean": -0.07362565445026178,
|
|
"eval_rewards/reward_fn/std": 0.503447853899959,
|
|
"eval_runtime": 1706.6586,
|
|
"eval_samples_per_second": 0.336,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.277582466082215,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000125297701172,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7611577148836944,
|
|
"eval_sampling/sampling_logp_difference/max": 0.3050420353966026,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021830966759531598,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.6,
|
|
"completions/max_terminated_length": 287.6,
|
|
"completions/mean_length": 204.3125,
|
|
"completions/mean_terminated_length": 204.3125,
|
|
"completions/min_length": 136.4,
|
|
"completions/min_terminated_length": 136.4,
|
|
"entropy": 1.059968823939562,
|
|
"epoch": 0.056081640158131836,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.471683502197266,
|
|
"kl": 0.0012696869809587952,
|
|
"learning_rate": 1.865808823529412e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 2663513.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.5180439412593841,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.5180439591407776,
|
|
"sampling/importance_sampling_ratio/max": 1.2942311406135558,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998626291751862,
|
|
"sampling/importance_sampling_ratio/min": 0.7758565127849579,
|
|
"sampling/sampling_logp_difference/max": 0.2907403588294983,
|
|
"sampling/sampling_logp_difference/mean": 0.021082007698714732,
|
|
"step": 610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.6,
|
|
"completions/max_terminated_length": 324.6,
|
|
"completions/mean_length": 221.0,
|
|
"completions/mean_terminated_length": 221.0,
|
|
"completions/min_length": 127.9,
|
|
"completions/min_terminated_length": 127.9,
|
|
"entropy": 1.1456563584506512,
|
|
"epoch": 0.05700101130826515,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.6595869064331055,
|
|
"kl": 0.0012122080097469734,
|
|
"learning_rate": 1.8964460784313725e-06,
|
|
"loss": -0.0004,
|
|
"num_tokens": 2711577.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.7250636041164398,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.7250636458396912,
|
|
"sampling/importance_sampling_ratio/max": 1.2696220874786377,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004298686981201,
|
|
"sampling/importance_sampling_ratio/min": 0.7179760634899139,
|
|
"sampling/sampling_logp_difference/max": 0.351926326751709,
|
|
"sampling/sampling_logp_difference/mean": 0.022937136143445967,
|
|
"step": 620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 334.4,
|
|
"completions/max_terminated_length": 319.9,
|
|
"completions/mean_length": 226.2375,
|
|
"completions/mean_terminated_length": 223.80178833007812,
|
|
"completions/min_length": 117.0,
|
|
"completions/min_terminated_length": 117.0,
|
|
"entropy": 1.134413019567728,
|
|
"epoch": 0.05792038245839846,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.442356586456299,
|
|
"kl": 0.0011642233301245141,
|
|
"learning_rate": 1.9270833333333334e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 2756196.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.4039553329348564,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.40395533889532087,
|
|
"sampling/importance_sampling_ratio/max": 1.2688979506492615,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997282147407531,
|
|
"sampling/importance_sampling_ratio/min": 0.7676404356956482,
|
|
"sampling/sampling_logp_difference/max": 0.2864227324724197,
|
|
"sampling/sampling_logp_difference/mean": 0.022699452936649323,
|
|
"step": 630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 311.0,
|
|
"completions/max_terminated_length": 307.8,
|
|
"completions/mean_length": 224.5625,
|
|
"completions/mean_terminated_length": 222.375,
|
|
"completions/min_length": 153.5,
|
|
"completions/min_terminated_length": 153.5,
|
|
"entropy": 1.1004840791225434,
|
|
"epoch": 0.05883975360853176,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.381046772003174,
|
|
"kl": 0.0013143535448762122,
|
|
"learning_rate": 1.9577205882352944e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 2804593.0,
|
|
"reward": 0.00625,
|
|
"reward_std": 0.4376678436994553,
|
|
"rewards/reward_fn/mean": 0.00625,
|
|
"rewards/reward_fn/std": 0.43766787350177766,
|
|
"sampling/importance_sampling_ratio/max": 1.2596550703048706,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996941030025482,
|
|
"sampling/importance_sampling_ratio/min": 0.7166162163019181,
|
|
"sampling/sampling_logp_difference/max": 0.3541571795940399,
|
|
"sampling/sampling_logp_difference/mean": 0.022227655909955502,
|
|
"step": 640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.6,
|
|
"completions/max_terminated_length": 281.6,
|
|
"completions/mean_length": 200.475,
|
|
"completions/mean_terminated_length": 200.475,
|
|
"completions/min_length": 137.4,
|
|
"completions/min_terminated_length": 137.4,
|
|
"entropy": 0.9952188201248646,
|
|
"epoch": 0.05975912475866507,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.908881187438965,
|
|
"kl": 0.0011727314042218496,
|
|
"learning_rate": 1.988357843137255e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 2848967.0,
|
|
"reward": -0.3375,
|
|
"reward_std": 0.49928959012031554,
|
|
"rewards/reward_fn/mean": -0.3375,
|
|
"rewards/reward_fn/std": 0.49928959608078005,
|
|
"sampling/importance_sampling_ratio/max": 1.3279521346092225,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001044511795043,
|
|
"sampling/importance_sampling_ratio/min": 0.7699520707130432,
|
|
"sampling/sampling_logp_difference/max": 0.3227164030075073,
|
|
"sampling/sampling_logp_difference/mean": 0.020919199753552675,
|
|
"step": 650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 266.7,
|
|
"completions/max_terminated_length": 266.7,
|
|
"completions/mean_length": 202.9625,
|
|
"completions/mean_terminated_length": 202.9625,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 1.0741405218839646,
|
|
"epoch": 0.06067849590879838,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.46884822845459,
|
|
"kl": 0.0013214543490903453,
|
|
"learning_rate": 2.018995098039216e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 2894356.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.46570733189582825,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.4657073378562927,
|
|
"sampling/importance_sampling_ratio/max": 1.2397753715515136,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001684188842774,
|
|
"sampling/importance_sampling_ratio/min": 0.7771768510341645,
|
|
"sampling/sampling_logp_difference/max": 0.2563755869865417,
|
|
"sampling/sampling_logp_difference/mean": 0.021589200757443906,
|
|
"step": 660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 328.5,
|
|
"completions/max_terminated_length": 328.5,
|
|
"completions/mean_length": 220.1625,
|
|
"completions/mean_terminated_length": 220.1625,
|
|
"completions/min_length": 136.7,
|
|
"completions/min_terminated_length": 136.7,
|
|
"entropy": 1.1027231484651565,
|
|
"epoch": 0.06159786705893169,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.490810871124268,
|
|
"kl": 0.0015161238821747246,
|
|
"learning_rate": 2.0496323529411767e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 2937417.0,
|
|
"reward": -0.4,
|
|
"reward_std": 0.8197159588336944,
|
|
"rewards/reward_fn/mean": -0.4,
|
|
"rewards/reward_fn/std": 0.8197160124778747,
|
|
"sampling/importance_sampling_ratio/max": 1.3400521159172059,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001618027687074,
|
|
"sampling/importance_sampling_ratio/min": 0.7698067843914032,
|
|
"sampling/sampling_logp_difference/max": 0.30671982765197753,
|
|
"sampling/sampling_logp_difference/mean": 0.02209597863256931,
|
|
"step": 670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 342.7,
|
|
"completions/max_terminated_length": 306.3,
|
|
"completions/mean_length": 228.9625,
|
|
"completions/mean_terminated_length": 222.5482162475586,
|
|
"completions/min_length": 151.0,
|
|
"completions/min_terminated_length": 151.0,
|
|
"entropy": 1.0638794735074044,
|
|
"epoch": 0.062517238209065,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.243893623352051,
|
|
"kl": 0.0018352417064306792,
|
|
"learning_rate": 2.0802696078431376e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 2985102.0,
|
|
"reward": 0.05625,
|
|
"reward_std": 0.5753691732883454,
|
|
"rewards/reward_fn/mean": 0.05625,
|
|
"rewards/reward_fn/std": 0.5753691852092743,
|
|
"sampling/importance_sampling_ratio/max": 1.3427037000656128,
|
|
"sampling/importance_sampling_ratio/mean": 0.999526858329773,
|
|
"sampling/importance_sampling_ratio/min": 0.7472610712051392,
|
|
"sampling/sampling_logp_difference/max": 0.3482950508594513,
|
|
"sampling/sampling_logp_difference/mean": 0.022305504232645035,
|
|
"step": 680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.6,
|
|
"completions/max_terminated_length": 314.6,
|
|
"completions/mean_length": 208.5125,
|
|
"completions/mean_terminated_length": 208.5125,
|
|
"completions/min_length": 131.2,
|
|
"completions/min_terminated_length": 131.2,
|
|
"entropy": 1.0907903552055358,
|
|
"epoch": 0.06343660935919831,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1277386099100113,
|
|
"kl": 0.0019939201170927844,
|
|
"learning_rate": 2.110906862745098e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 3030775.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.6294712632894516,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.6294712632894516,
|
|
"sampling/importance_sampling_ratio/max": 1.2345026850700378,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001937687397002,
|
|
"sampling/importance_sampling_ratio/min": 0.7667145550251007,
|
|
"sampling/sampling_logp_difference/max": 0.2702470779418945,
|
|
"sampling/sampling_logp_difference/mean": 0.022137697972357272,
|
|
"step": 690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 321.5,
|
|
"completions/max_terminated_length": 321.5,
|
|
"completions/mean_length": 226.5,
|
|
"completions/mean_terminated_length": 226.5,
|
|
"completions/min_length": 144.2,
|
|
"completions/min_terminated_length": 144.2,
|
|
"entropy": 1.0019915945827962,
|
|
"epoch": 0.06435598050933161,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.810304641723633,
|
|
"kl": 0.0017923590748978313,
|
|
"learning_rate": 2.141544117647059e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 3075815.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.7033166900277138,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.7033166900277138,
|
|
"sampling/importance_sampling_ratio/max": 1.3033283948898315,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996226727962494,
|
|
"sampling/importance_sampling_ratio/min": 0.7687257885932922,
|
|
"sampling/sampling_logp_difference/max": 0.30762053430080416,
|
|
"sampling/sampling_logp_difference/mean": 0.020816490799188615,
|
|
"step": 700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 263.1,
|
|
"completions/max_terminated_length": 263.1,
|
|
"completions/mean_length": 181.625,
|
|
"completions/mean_terminated_length": 181.625,
|
|
"completions/min_length": 114.2,
|
|
"completions/min_terminated_length": 114.2,
|
|
"entropy": 1.052776575088501,
|
|
"epoch": 0.06527535165946492,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.5576653480529785,
|
|
"kl": 0.0019860038912156595,
|
|
"learning_rate": 2.17218137254902e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 3116377.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.5737014353275299,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.5737014710903168,
|
|
"sampling/importance_sampling_ratio/max": 1.2989729404449464,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002965211868287,
|
|
"sampling/importance_sampling_ratio/min": 0.7976649224758148,
|
|
"sampling/sampling_logp_difference/max": 0.26987260580062866,
|
|
"sampling/sampling_logp_difference/mean": 0.02193618509918451,
|
|
"step": 710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.8,
|
|
"completions/max_terminated_length": 305.8,
|
|
"completions/mean_length": 214.275,
|
|
"completions/mean_terminated_length": 214.275,
|
|
"completions/min_length": 144.7,
|
|
"completions/min_terminated_length": 144.7,
|
|
"entropy": 1.0446342997252942,
|
|
"epoch": 0.06619472280959823,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.06392182409763336,
|
|
"kl": 0.0020218260469846427,
|
|
"learning_rate": 2.2028186274509805e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 3160351.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.5578714311122894,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.5578714430332183,
|
|
"sampling/importance_sampling_ratio/max": 1.3183648705482482,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995230078697205,
|
|
"sampling/importance_sampling_ratio/min": 0.7686205208301544,
|
|
"sampling/sampling_logp_difference/max": 0.31160278916358947,
|
|
"sampling/sampling_logp_difference/mean": 0.02191967945545912,
|
|
"step": 720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.2,
|
|
"completions/max_terminated_length": 308.2,
|
|
"completions/mean_length": 213.0125,
|
|
"completions/mean_terminated_length": 213.0125,
|
|
"completions/min_length": 143.8,
|
|
"completions/min_terminated_length": 143.8,
|
|
"entropy": 1.016400897502899,
|
|
"epoch": 0.06711409395973154,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.154075622558594,
|
|
"kl": 0.001864359564933693,
|
|
"learning_rate": 2.2334558823529414e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 3205568.0,
|
|
"reward": -0.475,
|
|
"reward_std": 0.5671322226524353,
|
|
"rewards/reward_fn/mean": -0.475,
|
|
"rewards/reward_fn/std": 0.5671322464942932,
|
|
"sampling/importance_sampling_ratio/max": 1.231548273563385,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002602279186248,
|
|
"sampling/importance_sampling_ratio/min": 0.7503688395023346,
|
|
"sampling/sampling_logp_difference/max": 0.2985227793455124,
|
|
"sampling/sampling_logp_difference/mean": 0.02064361944794655,
|
|
"step": 730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.0,
|
|
"completions/max_terminated_length": 285.0,
|
|
"completions/mean_length": 198.6875,
|
|
"completions/mean_terminated_length": 198.6875,
|
|
"completions/min_length": 123.5,
|
|
"completions/min_terminated_length": 123.5,
|
|
"entropy": 1.044501344114542,
|
|
"epoch": 0.06803346510986485,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.10327262431383133,
|
|
"kl": 0.0023841019843530375,
|
|
"learning_rate": 2.2640931372549023e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 3247447.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.4298454225063324,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.42984544634819033,
|
|
"sampling/importance_sampling_ratio/max": 1.228196370601654,
|
|
"sampling/importance_sampling_ratio/mean": 1.000384110212326,
|
|
"sampling/importance_sampling_ratio/min": 0.7776436030864715,
|
|
"sampling/sampling_logp_difference/max": 0.2547809600830078,
|
|
"sampling/sampling_logp_difference/mean": 0.020780765824019908,
|
|
"step": 740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.6,
|
|
"completions/max_terminated_length": 284.6,
|
|
"completions/mean_length": 199.3375,
|
|
"completions/mean_terminated_length": 199.3375,
|
|
"completions/min_length": 125.8,
|
|
"completions/min_terminated_length": 125.8,
|
|
"entropy": 1.0376607328653336,
|
|
"epoch": 0.06895283625999817,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.038123052567243576,
|
|
"kl": 0.0029976150144648273,
|
|
"learning_rate": 2.294730392156863e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 3286538.0,
|
|
"reward": 0.00625,
|
|
"reward_std": 0.37166147828102114,
|
|
"rewards/reward_fn/mean": 0.00625,
|
|
"rewards/reward_fn/std": 0.37166147828102114,
|
|
"sampling/importance_sampling_ratio/max": 1.2747241973876953,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998388707637786,
|
|
"sampling/importance_sampling_ratio/min": 0.7131896018981934,
|
|
"sampling/sampling_logp_difference/max": 0.3752714037895203,
|
|
"sampling/sampling_logp_difference/mean": 0.021647820994257928,
|
|
"step": 750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.4,
|
|
"completions/max_terminated_length": 308.4,
|
|
"completions/mean_length": 210.375,
|
|
"completions/mean_terminated_length": 210.375,
|
|
"completions/min_length": 138.8,
|
|
"completions/min_terminated_length": 138.8,
|
|
"entropy": 1.0729804262518883,
|
|
"epoch": 0.06987220741013148,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.930842876434326,
|
|
"kl": 0.002814591173955705,
|
|
"learning_rate": 2.3253676470588237e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 3330480.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.5096566110849381,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.5096566289663315,
|
|
"sampling/importance_sampling_ratio/max": 1.2650708317756654,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998493313789367,
|
|
"sampling/importance_sampling_ratio/min": 0.7998102128505706,
|
|
"sampling/sampling_logp_difference/max": 0.2504520356655121,
|
|
"sampling/sampling_logp_difference/mean": 0.02120122741907835,
|
|
"step": 760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.9,
|
|
"completions/max_terminated_length": 291.9,
|
|
"completions/mean_length": 212.1125,
|
|
"completions/mean_terminated_length": 212.1125,
|
|
"completions/min_length": 127.0,
|
|
"completions/min_terminated_length": 127.0,
|
|
"entropy": 1.0424277201294898,
|
|
"epoch": 0.07079157856026477,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.779900074005127,
|
|
"kl": 0.0019695476890774443,
|
|
"learning_rate": 2.3560049019607847e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 3374473.0,
|
|
"reward": -0.425,
|
|
"reward_std": 0.5967825591564179,
|
|
"rewards/reward_fn/mean": -0.425,
|
|
"rewards/reward_fn/std": 0.5967825591564179,
|
|
"sampling/importance_sampling_ratio/max": 1.2580410718917847,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996964335441589,
|
|
"sampling/importance_sampling_ratio/min": 0.7536938577890396,
|
|
"sampling/sampling_logp_difference/max": 0.3311957329511642,
|
|
"sampling/sampling_logp_difference/mean": 0.021131057664752005,
|
|
"step": 770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 306.8,
|
|
"completions/max_terminated_length": 302.1,
|
|
"completions/mean_length": 214.1,
|
|
"completions/mean_terminated_length": 211.575,
|
|
"completions/min_length": 129.6,
|
|
"completions/min_terminated_length": 129.6,
|
|
"entropy": 1.0404712975025177,
|
|
"epoch": 0.07171094971039808,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.05750023201107979,
|
|
"kl": 0.003591870867239777,
|
|
"learning_rate": 2.3866421568627456e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 3415681.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.4439386010169983,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.44393863081932067,
|
|
"sampling/importance_sampling_ratio/max": 1.3732612013816834,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003301322460174,
|
|
"sampling/importance_sampling_ratio/min": 0.7330769240856171,
|
|
"sampling/sampling_logp_difference/max": 0.38779781460762025,
|
|
"sampling/sampling_logp_difference/mean": 0.021037154272198676,
|
|
"step": 780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.6,
|
|
"completions/max_terminated_length": 272.6,
|
|
"completions/mean_length": 202.5125,
|
|
"completions/mean_terminated_length": 202.5125,
|
|
"completions/min_length": 143.3,
|
|
"completions/min_terminated_length": 143.3,
|
|
"entropy": 1.05287319496274,
|
|
"epoch": 0.0726303208605314,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.309450626373291,
|
|
"kl": 0.002909520715184044,
|
|
"learning_rate": 2.417279411764706e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 3457842.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.5087977588176728,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.5087977826595307,
|
|
"sampling/importance_sampling_ratio/max": 1.2744810581207275,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999706447124481,
|
|
"sampling/importance_sampling_ratio/min": 0.7788361012935638,
|
|
"sampling/sampling_logp_difference/max": 0.2818069696426392,
|
|
"sampling/sampling_logp_difference/mean": 0.021466401033103467,
|
|
"step": 790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.1,
|
|
"completions/max_terminated_length": 286.1,
|
|
"completions/mean_length": 200.7625,
|
|
"completions/mean_terminated_length": 200.7625,
|
|
"completions/min_length": 140.6,
|
|
"completions/min_terminated_length": 140.6,
|
|
"entropy": 1.089126033335924,
|
|
"epoch": 0.0735496920106647,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 7.358144760131836,
|
|
"kl": 0.0025294901599409057,
|
|
"learning_rate": 2.4479166666666666e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 3500271.0,
|
|
"reward": -0.6375,
|
|
"reward_std": 0.39377824664115907,
|
|
"rewards/reward_fn/mean": -0.6375,
|
|
"rewards/reward_fn/std": 0.39377824664115907,
|
|
"sampling/importance_sampling_ratio/max": 1.2900829076766969,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997484087944031,
|
|
"sampling/importance_sampling_ratio/min": 0.7701646745204925,
|
|
"sampling/sampling_logp_difference/max": 0.29433541297912597,
|
|
"sampling/sampling_logp_difference/mean": 0.02190215401351452,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.0735496920106647,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.001963350785340314,
|
|
"eval_completions/max_length": 293.5549738219895,
|
|
"eval_completions/max_terminated_length": 291.27748691099475,
|
|
"eval_completions/mean_length": 204.33944153577661,
|
|
"eval_completions/mean_terminated_length": 203.83570209389998,
|
|
"eval_completions/min_length": 132.88830715532285,
|
|
"eval_completions/min_terminated_length": 132.88830715532285,
|
|
"eval_entropy": 1.0747235617088398,
|
|
"eval_frac_reward_zero_std": 0.33507853403141363,
|
|
"eval_kl": 0.003043472509462237,
|
|
"eval_loss": 0.00012255448382347822,
|
|
"eval_num_tokens": 3500271.0,
|
|
"eval_reward": -0.06489965095986039,
|
|
"eval_reward_std": 0.5076892879152797,
|
|
"eval_rewards/reward_fn/mean": -0.06489965095986039,
|
|
"eval_rewards/reward_fn/std": 0.5076893038306561,
|
|
"eval_runtime": 1714.5099,
|
|
"eval_samples_per_second": 0.334,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2751857127610717,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000065688895514,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7604119822148877,
|
|
"eval_sampling/sampling_logp_difference/max": 0.30892977253720816,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02196653231971915,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.4,
|
|
"completions/max_terminated_length": 326.4,
|
|
"completions/mean_length": 210.1,
|
|
"completions/mean_terminated_length": 210.1,
|
|
"completions/min_length": 133.9,
|
|
"completions/min_terminated_length": 133.9,
|
|
"entropy": 1.0941917210817338,
|
|
"epoch": 0.07446906316079802,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0468394011259079,
|
|
"kl": 0.003635933337500319,
|
|
"learning_rate": 2.478553921568628e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 3547407.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.4316474676132202,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.43164747953414917,
|
|
"sampling/importance_sampling_ratio/max": 1.2295541405677795,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998320877552033,
|
|
"sampling/importance_sampling_ratio/min": 0.7933319866657257,
|
|
"sampling/sampling_logp_difference/max": 0.23681988716125488,
|
|
"sampling/sampling_logp_difference/mean": 0.022012897953391074,
|
|
"step": 810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.7,
|
|
"completions/max_terminated_length": 308.7,
|
|
"completions/mean_length": 210.875,
|
|
"completions/mean_terminated_length": 210.875,
|
|
"completions/min_length": 122.1,
|
|
"completions/min_terminated_length": 122.1,
|
|
"entropy": 1.0059361286461352,
|
|
"epoch": 0.07538843431093133,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.04860125854611397,
|
|
"kl": 0.002773964193329448,
|
|
"learning_rate": 2.5091911764705884e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 3591517.0,
|
|
"reward": -0.01875,
|
|
"reward_std": 0.566309529542923,
|
|
"rewards/reward_fn/mean": -0.01875,
|
|
"rewards/reward_fn/std": 0.5663095355033875,
|
|
"sampling/importance_sampling_ratio/max": 1.2595862746238708,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000875651836396,
|
|
"sampling/importance_sampling_ratio/min": 0.7891337037086487,
|
|
"sampling/sampling_logp_difference/max": 0.25611660480499265,
|
|
"sampling/sampling_logp_difference/mean": 0.020061161369085312,
|
|
"step": 820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 307.9,
|
|
"completions/max_terminated_length": 289.1,
|
|
"completions/mean_length": 207.45,
|
|
"completions/mean_terminated_length": 203.99285888671875,
|
|
"completions/min_length": 137.7,
|
|
"completions/min_terminated_length": 137.7,
|
|
"entropy": 1.060875914245844,
|
|
"epoch": 0.07630780546106464,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.04882955551147461,
|
|
"kl": 0.004919398497440853,
|
|
"learning_rate": 2.539828431372549e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 3633353.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.5631370812654495,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.5631370931863785,
|
|
"sampling/importance_sampling_ratio/max": 1.2809331178665162,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995364725589753,
|
|
"sampling/importance_sampling_ratio/min": 0.7481603860855103,
|
|
"sampling/sampling_logp_difference/max": 0.3214196145534515,
|
|
"sampling/sampling_logp_difference/mean": 0.02176289279013872,
|
|
"step": 830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.3,
|
|
"completions/max_terminated_length": 292.3,
|
|
"completions/mean_length": 194.4375,
|
|
"completions/mean_terminated_length": 194.4375,
|
|
"completions/min_length": 118.1,
|
|
"completions/min_terminated_length": 118.1,
|
|
"entropy": 1.0361232250928878,
|
|
"epoch": 0.07722717661119793,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.146517276763916,
|
|
"kl": 0.0029901799985964316,
|
|
"learning_rate": 2.57046568627451e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 3674556.0,
|
|
"reward": -0.5125,
|
|
"reward_std": 0.4509626179933548,
|
|
"rewards/reward_fn/mean": -0.5125,
|
|
"rewards/reward_fn/std": 0.45096264183521273,
|
|
"sampling/importance_sampling_ratio/max": 1.2726532220840454,
|
|
"sampling/importance_sampling_ratio/mean": 0.9992108643054962,
|
|
"sampling/importance_sampling_ratio/min": 0.7762696444988251,
|
|
"sampling/sampling_logp_difference/max": 0.2775296330451965,
|
|
"sampling/sampling_logp_difference/mean": 0.02096966076642275,
|
|
"step": 840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.3,
|
|
"completions/max_terminated_length": 298.3,
|
|
"completions/mean_length": 201.3,
|
|
"completions/mean_terminated_length": 201.3,
|
|
"completions/min_length": 129.2,
|
|
"completions/min_terminated_length": 129.2,
|
|
"entropy": 1.1129839710891247,
|
|
"epoch": 0.07814654776133124,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.1760735511779785,
|
|
"kl": 0.003231057215452893,
|
|
"learning_rate": 2.601102941176471e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 3719852.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.5990100085735321,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.5990100562572479,
|
|
"sampling/importance_sampling_ratio/max": 1.317863154411316,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996878027915954,
|
|
"sampling/importance_sampling_ratio/min": 0.7930496752262115,
|
|
"sampling/sampling_logp_difference/max": 0.2882010996341705,
|
|
"sampling/sampling_logp_difference/mean": 0.021463465876877308,
|
|
"step": 850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 249.5,
|
|
"completions/max_terminated_length": 249.5,
|
|
"completions/mean_length": 187.05,
|
|
"completions/mean_terminated_length": 187.05,
|
|
"completions/min_length": 131.2,
|
|
"completions/min_terminated_length": 131.2,
|
|
"entropy": 1.0520827293395996,
|
|
"epoch": 0.07906591891146456,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0969572439789772,
|
|
"kl": 0.0027062737382948397,
|
|
"learning_rate": 2.6317401960784317e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 3758776.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.4006543099880219,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.4006543219089508,
|
|
"sampling/importance_sampling_ratio/max": 1.2775512337684631,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999620974063873,
|
|
"sampling/importance_sampling_ratio/min": 0.7652165651321411,
|
|
"sampling/sampling_logp_difference/max": 0.288371616601944,
|
|
"sampling/sampling_logp_difference/mean": 0.021277916617691517,
|
|
"step": 860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 326.4,
|
|
"completions/max_terminated_length": 315.8,
|
|
"completions/mean_length": 218.25,
|
|
"completions/mean_terminated_length": 215.00892944335936,
|
|
"completions/min_length": 140.4,
|
|
"completions/min_terminated_length": 140.4,
|
|
"entropy": 1.1387375429272653,
|
|
"epoch": 0.07998529006159787,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.6979475021362305,
|
|
"kl": 0.002763241647335235,
|
|
"learning_rate": 2.662377450980392e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 3804252.0,
|
|
"reward": 0.18125,
|
|
"reward_std": 0.41367453932762144,
|
|
"rewards/reward_fn/mean": 0.18125,
|
|
"rewards/reward_fn/std": 0.4136745512485504,
|
|
"sampling/importance_sampling_ratio/max": 1.2411170601844788,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003732740879059,
|
|
"sampling/importance_sampling_ratio/min": 0.7765689730644226,
|
|
"sampling/sampling_logp_difference/max": 0.25886437892913816,
|
|
"sampling/sampling_logp_difference/mean": 0.021854002587497236,
|
|
"step": 870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 267.5,
|
|
"completions/max_terminated_length": 267.5,
|
|
"completions/mean_length": 192.725,
|
|
"completions/mean_terminated_length": 192.725,
|
|
"completions/min_length": 119.7,
|
|
"completions/min_terminated_length": 119.7,
|
|
"entropy": 0.9888180930167436,
|
|
"epoch": 0.08090466121173118,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.668768405914307,
|
|
"kl": 0.0038015075013390744,
|
|
"learning_rate": 2.693014705882353e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 3847798.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.41135978102684023,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.4113598048686981,
|
|
"sampling/importance_sampling_ratio/max": 1.2296013355255127,
|
|
"sampling/importance_sampling_ratio/mean": 0.9992747724056243,
|
|
"sampling/importance_sampling_ratio/min": 0.7497190058231353,
|
|
"sampling/sampling_logp_difference/max": 0.3144635081291199,
|
|
"sampling/sampling_logp_difference/mean": 0.020920691825449467,
|
|
"step": 880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.6,
|
|
"completions/max_terminated_length": 271.6,
|
|
"completions/mean_length": 185.275,
|
|
"completions/mean_terminated_length": 185.275,
|
|
"completions/min_length": 115.9,
|
|
"completions/min_terminated_length": 115.9,
|
|
"entropy": 0.9807270668447018,
|
|
"epoch": 0.08182403236186449,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.880890369415283,
|
|
"kl": 0.0029675340076209976,
|
|
"learning_rate": 2.7236519607843136e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 3887212.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.4339252322912216,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.4339252501726151,
|
|
"sampling/importance_sampling_ratio/max": 1.2872227907180787,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002362549304962,
|
|
"sampling/importance_sampling_ratio/min": 0.7751561105251312,
|
|
"sampling/sampling_logp_difference/max": 0.3027405202388763,
|
|
"sampling/sampling_logp_difference/mean": 0.02112089265137911,
|
|
"step": 890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.4,
|
|
"completions/max_terminated_length": 283.4,
|
|
"completions/mean_length": 197.3625,
|
|
"completions/mean_terminated_length": 197.3625,
|
|
"completions/min_length": 112.8,
|
|
"completions/min_terminated_length": 112.8,
|
|
"entropy": 1.0829822316765785,
|
|
"epoch": 0.0827434035119978,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.487550258636475,
|
|
"kl": 0.007837075377756264,
|
|
"learning_rate": 2.754289215686275e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 3928489.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.5465950816869736,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.5465950936079025,
|
|
"sampling/importance_sampling_ratio/max": 1.346196746826172,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996935188770294,
|
|
"sampling/importance_sampling_ratio/min": 0.7384186804294586,
|
|
"sampling/sampling_logp_difference/max": 0.36049288511276245,
|
|
"sampling/sampling_logp_difference/mean": 0.02240522187203169,
|
|
"step": 900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.1,
|
|
"completions/max_terminated_length": 273.1,
|
|
"completions/mean_length": 191.075,
|
|
"completions/mean_terminated_length": 191.075,
|
|
"completions/min_length": 126.2,
|
|
"completions/min_terminated_length": 126.2,
|
|
"entropy": 0.9956984333693981,
|
|
"epoch": 0.08366277466213111,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.19687032699585,
|
|
"kl": 0.006185506476322189,
|
|
"learning_rate": 2.7849264705882355e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 3968975.0,
|
|
"reward": -0.375,
|
|
"reward_std": 0.5235352247953415,
|
|
"rewards/reward_fn/mean": -0.375,
|
|
"rewards/reward_fn/std": 0.5235352486371994,
|
|
"sampling/importance_sampling_ratio/max": 1.2452479362487794,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005298733711243,
|
|
"sampling/importance_sampling_ratio/min": 0.7847002863883972,
|
|
"sampling/sampling_logp_difference/max": 0.250644251704216,
|
|
"sampling/sampling_logp_difference/mean": 0.021063000336289404,
|
|
"step": 910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.0,
|
|
"completions/max_terminated_length": 286.0,
|
|
"completions/mean_length": 193.075,
|
|
"completions/mean_terminated_length": 193.075,
|
|
"completions/min_length": 121.4,
|
|
"completions/min_terminated_length": 121.4,
|
|
"entropy": 0.998225849866867,
|
|
"epoch": 0.0845821458122644,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 9.962566375732422,
|
|
"kl": 0.006246131453372073,
|
|
"learning_rate": 2.8155637254901964e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 4009973.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.7299546062946319,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.7299546241760254,
|
|
"sampling/importance_sampling_ratio/max": 1.2650769233703614,
|
|
"sampling/importance_sampling_ratio/mean": 1.000108653306961,
|
|
"sampling/importance_sampling_ratio/min": 0.7777700006961823,
|
|
"sampling/sampling_logp_difference/max": 0.26809492111206057,
|
|
"sampling/sampling_logp_difference/mean": 0.021244184114038944,
|
|
"step": 920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.0,
|
|
"completions/max_terminated_length": 319.0,
|
|
"completions/mean_length": 221.25,
|
|
"completions/mean_terminated_length": 221.25,
|
|
"completions/min_length": 141.9,
|
|
"completions/min_terminated_length": 141.9,
|
|
"entropy": 1.0295479901134967,
|
|
"epoch": 0.08550151696239772,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.537407875061035,
|
|
"kl": 0.006818726073834114,
|
|
"learning_rate": 2.846200980392157e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 4056449.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.5779510378837586,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.5779510498046875,
|
|
"sampling/importance_sampling_ratio/max": 1.317081832885742,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995251536369324,
|
|
"sampling/importance_sampling_ratio/min": 0.7700926482677459,
|
|
"sampling/sampling_logp_difference/max": 0.30742167532444,
|
|
"sampling/sampling_logp_difference/mean": 0.02112684603780508,
|
|
"step": 930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.9,
|
|
"completions/max_terminated_length": 285.9,
|
|
"completions/mean_length": 203.975,
|
|
"completions/mean_terminated_length": 203.975,
|
|
"completions/min_length": 126.1,
|
|
"completions/min_terminated_length": 126.1,
|
|
"entropy": 1.0224198609590531,
|
|
"epoch": 0.08642088811253103,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.340763568878174,
|
|
"kl": 0.0062911443543271165,
|
|
"learning_rate": 2.876838235294118e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 4096711.0,
|
|
"reward": -0.05625,
|
|
"reward_std": 0.606641286611557,
|
|
"rewards/reward_fn/mean": -0.05625,
|
|
"rewards/reward_fn/std": 0.6066413044929504,
|
|
"sampling/importance_sampling_ratio/max": 1.2410983204841615,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997718155384063,
|
|
"sampling/importance_sampling_ratio/min": 0.7799016654491424,
|
|
"sampling/sampling_logp_difference/max": 0.25803680419921876,
|
|
"sampling/sampling_logp_difference/mean": 0.02161556761711836,
|
|
"step": 940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.5,
|
|
"completions/max_terminated_length": 308.5,
|
|
"completions/mean_length": 211.025,
|
|
"completions/mean_terminated_length": 211.025,
|
|
"completions/min_length": 126.8,
|
|
"completions/min_terminated_length": 126.8,
|
|
"entropy": 1.0685635283589363,
|
|
"epoch": 0.08734025926266434,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.713547229766846,
|
|
"kl": 0.006182058043486904,
|
|
"learning_rate": 2.9074754901960787e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 4139513.0,
|
|
"reward": 0.00625,
|
|
"reward_std": 0.5506034731864929,
|
|
"rewards/reward_fn/mean": 0.00625,
|
|
"rewards/reward_fn/std": 0.5506034791469574,
|
|
"sampling/importance_sampling_ratio/max": 1.376935601234436,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003678441047668,
|
|
"sampling/importance_sampling_ratio/min": 0.8011228561401367,
|
|
"sampling/sampling_logp_difference/max": 0.33361477255821226,
|
|
"sampling/sampling_logp_difference/mean": 0.02131375763565302,
|
|
"step": 950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 351.2,
|
|
"completions/max_terminated_length": 333.2,
|
|
"completions/mean_length": 242.9375,
|
|
"completions/mean_terminated_length": 239.1500015258789,
|
|
"completions/min_length": 153.6,
|
|
"completions/min_terminated_length": 153.6,
|
|
"entropy": 1.0816266678273678,
|
|
"epoch": 0.08825963041279765,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.561310768127441,
|
|
"kl": 0.005243503677775152,
|
|
"learning_rate": 2.9381127450980397e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 4184116.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.735353896021843,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.7353539139032363,
|
|
"sampling/importance_sampling_ratio/max": 1.2709896206855773,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998113334178924,
|
|
"sampling/importance_sampling_ratio/min": 0.7179039746522904,
|
|
"sampling/sampling_logp_difference/max": 0.3657096326351166,
|
|
"sampling/sampling_logp_difference/mean": 0.021757134050130845,
|
|
"step": 960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.8,
|
|
"completions/max_terminated_length": 319.8,
|
|
"completions/mean_length": 213.85,
|
|
"completions/mean_terminated_length": 213.85,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.0483902730047703,
|
|
"epoch": 0.08917900156293096,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.578107833862305,
|
|
"kl": 0.011414260690798983,
|
|
"learning_rate": 2.96875e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 4226344.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.49959076642990113,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.499590790271759,
|
|
"sampling/importance_sampling_ratio/max": 1.3289848446846009,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000871539115905,
|
|
"sampling/importance_sampling_ratio/min": 0.7711245894432068,
|
|
"sampling/sampling_logp_difference/max": 0.3512672662734985,
|
|
"sampling/sampling_logp_difference/mean": 0.020528460573405027,
|
|
"step": 970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.4,
|
|
"completions/max_terminated_length": 278.4,
|
|
"completions/mean_length": 197.9375,
|
|
"completions/mean_terminated_length": 197.9375,
|
|
"completions/min_length": 129.7,
|
|
"completions/min_terminated_length": 129.7,
|
|
"entropy": 1.0756870269775392,
|
|
"epoch": 0.09009837271306427,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.964056491851807,
|
|
"kl": 0.01016737746540457,
|
|
"learning_rate": 2.999387254901961e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 4267955.0,
|
|
"reward": -0.08125,
|
|
"reward_std": 0.4659160763025284,
|
|
"rewards/reward_fn/mean": -0.08125,
|
|
"rewards/reward_fn/std": 0.46591608226299286,
|
|
"sampling/importance_sampling_ratio/max": 1.2798233151435852,
|
|
"sampling/importance_sampling_ratio/mean": 1.000102198123932,
|
|
"sampling/importance_sampling_ratio/min": 0.587386866658926,
|
|
"sampling/sampling_logp_difference/max": 0.8602162718772888,
|
|
"sampling/sampling_logp_difference/mean": 0.021850606054067613,
|
|
"step": 980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.3,
|
|
"completions/max_terminated_length": 283.3,
|
|
"completions/mean_length": 190.0375,
|
|
"completions/mean_terminated_length": 190.0375,
|
|
"completions/min_length": 124.6,
|
|
"completions/min_terminated_length": 124.6,
|
|
"entropy": 1.049230270832777,
|
|
"epoch": 0.09101774386319757,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.506882190704346,
|
|
"kl": 0.010564835877448785,
|
|
"learning_rate": 3.0300245098039216e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 4307070.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.5135475039482117,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.5135475158691406,
|
|
"sampling/importance_sampling_ratio/max": 1.2762622237205505,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993451952934265,
|
|
"sampling/importance_sampling_ratio/min": 0.7719105541706085,
|
|
"sampling/sampling_logp_difference/max": 0.2895656704902649,
|
|
"sampling/sampling_logp_difference/mean": 0.021446413174271584,
|
|
"step": 990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.8,
|
|
"completions/max_terminated_length": 288.8,
|
|
"completions/mean_length": 200.35,
|
|
"completions/mean_terminated_length": 200.35,
|
|
"completions/min_length": 120.4,
|
|
"completions/min_terminated_length": 120.4,
|
|
"entropy": 1.0577165380120277,
|
|
"epoch": 0.09193711501333088,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.163789749145508,
|
|
"kl": 0.006241366888571065,
|
|
"learning_rate": 3.060661764705883e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 4349418.0,
|
|
"reward": -0.09375,
|
|
"reward_std": 0.6604974716901779,
|
|
"rewards/reward_fn/mean": -0.09375,
|
|
"rewards/reward_fn/std": 0.6604974955320358,
|
|
"sampling/importance_sampling_ratio/max": 1.2737963676452637,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003153443336488,
|
|
"sampling/importance_sampling_ratio/min": 0.7728460252285003,
|
|
"sampling/sampling_logp_difference/max": 0.2808604598045349,
|
|
"sampling/sampling_logp_difference/mean": 0.021331284753978254,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.09193711501333088,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002399650959860384,
|
|
"eval_completions/max_length": 288.0471204188482,
|
|
"eval_completions/max_terminated_length": 285.41012216404886,
|
|
"eval_completions/mean_length": 195.10362129144852,
|
|
"eval_completions/mean_terminated_length": 194.4758478933604,
|
|
"eval_completions/min_length": 123.08027923211169,
|
|
"eval_completions/min_terminated_length": 123.08027923211169,
|
|
"eval_entropy": 1.0918490369698557,
|
|
"eval_frac_reward_zero_std": 0.2879581151832461,
|
|
"eval_kl": 0.008752725989172596,
|
|
"eval_loss": 0.00039568814099766314,
|
|
"eval_num_tokens": 4349418.0,
|
|
"eval_reward": -0.04144851657940663,
|
|
"eval_reward_std": 0.49292232473275216,
|
|
"eval_rewards/reward_fn/mean": -0.04144851657940663,
|
|
"eval_rewards/reward_fn/std": 0.4929223387757313,
|
|
"eval_runtime": 1688.7144,
|
|
"eval_samples_per_second": 0.339,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2956086819500616,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999952431748674,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7587555581622515,
|
|
"eval_sampling/sampling_logp_difference/max": 0.3259408835965301,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021509873033883162,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.3,
|
|
"completions/max_terminated_length": 279.3,
|
|
"completions/mean_length": 204.9125,
|
|
"completions/mean_terminated_length": 204.9125,
|
|
"completions/min_length": 136.9,
|
|
"completions/min_terminated_length": 136.9,
|
|
"entropy": 1.1395965121686458,
|
|
"epoch": 0.09285648616346419,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.542971134185791,
|
|
"kl": 0.008696513864560985,
|
|
"learning_rate": 3.0912990196078434e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 4396227.0,
|
|
"reward": -0.275,
|
|
"reward_std": 0.6093845218420029,
|
|
"rewards/reward_fn/mean": -0.275,
|
|
"rewards/reward_fn/std": 0.6093845516443253,
|
|
"sampling/importance_sampling_ratio/max": 1.2921066522598266,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000555276870728,
|
|
"sampling/importance_sampling_ratio/min": 0.7674422830343246,
|
|
"sampling/sampling_logp_difference/max": 0.30101699829101564,
|
|
"sampling/sampling_logp_difference/mean": 0.022433127835392953,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.0,
|
|
"completions/max_terminated_length": 301.0,
|
|
"completions/mean_length": 206.125,
|
|
"completions/mean_terminated_length": 206.125,
|
|
"completions/min_length": 126.1,
|
|
"completions/min_terminated_length": 126.1,
|
|
"entropy": 1.1115202084183693,
|
|
"epoch": 0.0937758573135975,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.391073226928711,
|
|
"kl": 0.006414841805235483,
|
|
"learning_rate": 3.1219362745098044e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 4440333.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.535081484913826,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.5350814908742905,
|
|
"sampling/importance_sampling_ratio/max": 1.2689568161964417,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998329162597657,
|
|
"sampling/importance_sampling_ratio/min": 0.7038759469985962,
|
|
"sampling/sampling_logp_difference/max": 0.3774571418762207,
|
|
"sampling/sampling_logp_difference/mean": 0.02169222626835108,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 266.5,
|
|
"completions/max_terminated_length": 266.5,
|
|
"completions/mean_length": 194.575,
|
|
"completions/mean_terminated_length": 194.575,
|
|
"completions/min_length": 138.8,
|
|
"completions/min_terminated_length": 138.8,
|
|
"entropy": 1.0383225880563258,
|
|
"epoch": 0.09469522846373081,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.7692670822143555,
|
|
"kl": 0.01427452263305895,
|
|
"learning_rate": 3.152573529411765e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 4479491.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.6796805769205093,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.6796805888414383,
|
|
"sampling/importance_sampling_ratio/max": 1.2769641399383544,
|
|
"sampling/importance_sampling_ratio/mean": 1.000789326429367,
|
|
"sampling/importance_sampling_ratio/min": 0.7282934546470642,
|
|
"sampling/sampling_logp_difference/max": 0.352101719379425,
|
|
"sampling/sampling_logp_difference/mean": 0.020846835151314736,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.9,
|
|
"completions/max_terminated_length": 286.9,
|
|
"completions/mean_length": 205.9375,
|
|
"completions/mean_terminated_length": 205.9375,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 1.11657222956419,
|
|
"epoch": 0.09561459961386412,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.065827369689941,
|
|
"kl": 0.010999771510250866,
|
|
"learning_rate": 3.1832107843137254e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 4523246.0,
|
|
"reward": -0.375,
|
|
"reward_std": 0.5288968861103058,
|
|
"rewards/reward_fn/mean": -0.375,
|
|
"rewards/reward_fn/std": 0.5288969099521637,
|
|
"sampling/importance_sampling_ratio/max": 1.2682188630104065,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997571289539338,
|
|
"sampling/importance_sampling_ratio/min": 0.7869106948375701,
|
|
"sampling/sampling_logp_difference/max": 0.2651927322149277,
|
|
"sampling/sampling_logp_difference/mean": 0.02140223141759634,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 339.5,
|
|
"completions/max_terminated_length": 339.5,
|
|
"completions/mean_length": 221.075,
|
|
"completions/mean_terminated_length": 221.075,
|
|
"completions/min_length": 129.5,
|
|
"completions/min_terminated_length": 129.5,
|
|
"entropy": 1.0718603052198887,
|
|
"epoch": 0.09653397076399743,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.041465759277344,
|
|
"kl": 0.014499837646144442,
|
|
"learning_rate": 3.2138480392156867e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 4566860.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.5085761934518814,
|
|
"rewards/reward_fn/mean": 0.2875,
|
|
"rewards/reward_fn/std": 0.5085762113332748,
|
|
"sampling/importance_sampling_ratio/max": 1.2474358439445496,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997801959514618,
|
|
"sampling/importance_sampling_ratio/min": 0.7986530303955078,
|
|
"sampling/sampling_logp_difference/max": 0.24827976226806642,
|
|
"sampling/sampling_logp_difference/mean": 0.02116443943232298,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.0,
|
|
"completions/max_terminated_length": 325.0,
|
|
"completions/mean_length": 220.5125,
|
|
"completions/mean_terminated_length": 220.5125,
|
|
"completions/min_length": 142.6,
|
|
"completions/min_terminated_length": 142.6,
|
|
"entropy": 1.09085945263505,
|
|
"epoch": 0.09745334191413073,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.09769704937934875,
|
|
"kl": 0.01425217889191117,
|
|
"learning_rate": 3.2444852941176476e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 4611845.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.5213924765586853,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.5213924765586853,
|
|
"sampling/importance_sampling_ratio/max": 1.3713450193405152,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005628168582916,
|
|
"sampling/importance_sampling_ratio/min": 0.7301262199878693,
|
|
"sampling/sampling_logp_difference/max": 0.34665178060531615,
|
|
"sampling/sampling_logp_difference/mean": 0.021962456032633783,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.0,
|
|
"completions/max_terminated_length": 302.0,
|
|
"completions/mean_length": 201.025,
|
|
"completions/mean_terminated_length": 201.025,
|
|
"completions/min_length": 136.7,
|
|
"completions/min_terminated_length": 136.7,
|
|
"entropy": 1.209683421999216,
|
|
"epoch": 0.09837271306426404,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.830780982971191,
|
|
"kl": 0.012879665949731134,
|
|
"learning_rate": 3.275122549019608e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 4655087.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.5285374969244003,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.5285375028848648,
|
|
"sampling/importance_sampling_ratio/max": 1.3322084188461303,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994369089603424,
|
|
"sampling/importance_sampling_ratio/min": 0.7903971552848816,
|
|
"sampling/sampling_logp_difference/max": 0.3585920214653015,
|
|
"sampling/sampling_logp_difference/mean": 0.023745033517479898,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.9,
|
|
"completions/max_terminated_length": 326.9,
|
|
"completions/mean_length": 210.5375,
|
|
"completions/mean_terminated_length": 210.5375,
|
|
"completions/min_length": 130.2,
|
|
"completions/min_terminated_length": 130.2,
|
|
"entropy": 1.1410760872066021,
|
|
"epoch": 0.09929208421439735,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.08446750044822693,
|
|
"kl": 0.05767056857876014,
|
|
"learning_rate": 3.3057598039215686e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 4702522.0,
|
|
"reward": -0.24375,
|
|
"reward_std": 0.500537320971489,
|
|
"rewards/reward_fn/mean": -0.24375,
|
|
"rewards/reward_fn/std": 0.5005373388528824,
|
|
"sampling/importance_sampling_ratio/max": 1.2505094289779664,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000601351261138,
|
|
"sampling/importance_sampling_ratio/min": 0.7441504150629044,
|
|
"sampling/sampling_logp_difference/max": 0.3169679373502731,
|
|
"sampling/sampling_logp_difference/mean": 0.022094489261507987,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.2,
|
|
"completions/max_terminated_length": 283.2,
|
|
"completions/mean_length": 202.2375,
|
|
"completions/mean_terminated_length": 202.2375,
|
|
"completions/min_length": 139.6,
|
|
"completions/min_terminated_length": 139.6,
|
|
"entropy": 1.072489184141159,
|
|
"epoch": 0.10021145536453066,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.569357872009277,
|
|
"kl": 0.013545406749472022,
|
|
"learning_rate": 3.3363970588235295e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 4746397.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.5542894184589386,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.554289436340332,
|
|
"sampling/importance_sampling_ratio/max": 1.3320290684700011,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002446115016936,
|
|
"sampling/importance_sampling_ratio/min": 0.7941021025180817,
|
|
"sampling/sampling_logp_difference/max": 0.31126514077186584,
|
|
"sampling/sampling_logp_difference/mean": 0.021482798084616662,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.6,
|
|
"completions/max_terminated_length": 301.6,
|
|
"completions/mean_length": 192.825,
|
|
"completions/mean_terminated_length": 192.825,
|
|
"completions/min_length": 119.3,
|
|
"completions/min_terminated_length": 119.3,
|
|
"entropy": 1.0538658171892166,
|
|
"epoch": 0.10113082651466397,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.378344535827637,
|
|
"kl": 0.01226953813165892,
|
|
"learning_rate": 3.367034313725491e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 4787183.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.44131766855716703,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.44131766855716703,
|
|
"sampling/importance_sampling_ratio/max": 1.3314331650733948,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999584257602692,
|
|
"sampling/importance_sampling_ratio/min": 0.7500885725021362,
|
|
"sampling/sampling_logp_difference/max": 0.32559962272644044,
|
|
"sampling/sampling_logp_difference/mean": 0.021489018015563487,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.9,
|
|
"completions/max_terminated_length": 292.9,
|
|
"completions/mean_length": 199.3125,
|
|
"completions/mean_terminated_length": 199.3125,
|
|
"completions/min_length": 130.1,
|
|
"completions/min_terminated_length": 130.1,
|
|
"entropy": 1.0446888476610183,
|
|
"epoch": 0.10205019766479728,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.19413918256759644,
|
|
"kl": 0.012133742007426918,
|
|
"learning_rate": 3.3976715686274514e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 4833664.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.438364240527153,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.43836425244808197,
|
|
"sampling/importance_sampling_ratio/max": 1.2479475140571594,
|
|
"sampling/importance_sampling_ratio/mean": 1.000616192817688,
|
|
"sampling/importance_sampling_ratio/min": 0.7562075972557067,
|
|
"sampling/sampling_logp_difference/max": 0.2886667728424072,
|
|
"sampling/sampling_logp_difference/mean": 0.022036554664373396,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.8,
|
|
"completions/max_terminated_length": 300.8,
|
|
"completions/mean_length": 198.0875,
|
|
"completions/mean_terminated_length": 198.0875,
|
|
"completions/min_length": 128.9,
|
|
"completions/min_terminated_length": 128.9,
|
|
"entropy": 1.0662069909274579,
|
|
"epoch": 0.10296956881493059,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.773277282714844,
|
|
"kl": 0.007365902060701046,
|
|
"learning_rate": 3.428308823529412e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 4875367.0,
|
|
"reward": -0.3,
|
|
"reward_std": 0.42112618386745454,
|
|
"rewards/reward_fn/mean": -0.3,
|
|
"rewards/reward_fn/std": 0.421126189827919,
|
|
"sampling/importance_sampling_ratio/max": 1.2560564041137696,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996202647686004,
|
|
"sampling/importance_sampling_ratio/min": 0.7204268157482148,
|
|
"sampling/sampling_logp_difference/max": 0.3421809434890747,
|
|
"sampling/sampling_logp_difference/mean": 0.022097828239202498,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.5,
|
|
"completions/max_terminated_length": 295.5,
|
|
"completions/mean_length": 195.45,
|
|
"completions/mean_terminated_length": 195.45,
|
|
"completions/min_length": 118.5,
|
|
"completions/min_terminated_length": 118.5,
|
|
"entropy": 1.0936168320477009,
|
|
"epoch": 0.10388893996506389,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 8.33864974975586,
|
|
"kl": 0.01408274565183092,
|
|
"learning_rate": 3.458946078431373e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 4918427.0,
|
|
"reward": -0.31875,
|
|
"reward_std": 0.5171559393405915,
|
|
"rewards/reward_fn/mean": -0.31875,
|
|
"rewards/reward_fn/std": 0.5171559512615204,
|
|
"sampling/importance_sampling_ratio/max": 1.3096859097480773,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999238014221191,
|
|
"sampling/importance_sampling_ratio/min": 0.723172527551651,
|
|
"sampling/sampling_logp_difference/max": 0.35930281281471255,
|
|
"sampling/sampling_logp_difference/mean": 0.02284015081822872,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.5,
|
|
"completions/max_terminated_length": 309.5,
|
|
"completions/mean_length": 220.0625,
|
|
"completions/mean_terminated_length": 220.0625,
|
|
"completions/min_length": 134.3,
|
|
"completions/min_terminated_length": 134.3,
|
|
"entropy": 1.0693205066025258,
|
|
"epoch": 0.1048083111151972,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.8803791999816895,
|
|
"kl": 0.010877759483992121,
|
|
"learning_rate": 3.4895833333333333e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 4965360.0,
|
|
"reward": -0.325,
|
|
"reward_std": 0.5109704315662384,
|
|
"rewards/reward_fn/mean": -0.325,
|
|
"rewards/reward_fn/std": 0.5109704434871674,
|
|
"sampling/importance_sampling_ratio/max": 1.297501516342163,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997015535831452,
|
|
"sampling/importance_sampling_ratio/min": 0.7729672253131866,
|
|
"sampling/sampling_logp_difference/max": 0.29393945932388305,
|
|
"sampling/sampling_logp_difference/mean": 0.02267507053911686,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.6,
|
|
"completions/max_terminated_length": 296.6,
|
|
"completions/mean_length": 216.875,
|
|
"completions/mean_terminated_length": 216.875,
|
|
"completions/min_length": 141.9,
|
|
"completions/min_terminated_length": 141.9,
|
|
"entropy": 0.9997228812426329,
|
|
"epoch": 0.10572768226533051,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.133396625518799,
|
|
"kl": 0.013946997774473857,
|
|
"learning_rate": 3.5202205882352947e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 5007366.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.5775342971086502,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.5775343269109726,
|
|
"sampling/importance_sampling_ratio/max": 1.3933629989624023,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001138508319856,
|
|
"sampling/importance_sampling_ratio/min": 0.7607937216758728,
|
|
"sampling/sampling_logp_difference/max": 0.4101886212825775,
|
|
"sampling/sampling_logp_difference/mean": 0.021730512380599976,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 308.2,
|
|
"completions/max_terminated_length": 293.7,
|
|
"completions/mean_length": 208.8625,
|
|
"completions/mean_terminated_length": 205.3732147216797,
|
|
"completions/min_length": 132.9,
|
|
"completions/min_terminated_length": 132.9,
|
|
"entropy": 1.0656747043132782,
|
|
"epoch": 0.10664705341546382,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.143483638763428,
|
|
"kl": 0.010617410464328713,
|
|
"learning_rate": 3.550857843137255e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 5052755.0,
|
|
"reward": -0.05625,
|
|
"reward_std": 0.6774400025606155,
|
|
"rewards/reward_fn/mean": -0.05625,
|
|
"rewards/reward_fn/std": 0.6774400383234024,
|
|
"sampling/importance_sampling_ratio/max": 1.4161882400512695,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996443450450897,
|
|
"sampling/importance_sampling_ratio/min": 0.7458942472934723,
|
|
"sampling/sampling_logp_difference/max": 0.40879332423210146,
|
|
"sampling/sampling_logp_difference/mean": 0.022250896878540517,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.6,
|
|
"completions/max_terminated_length": 276.6,
|
|
"completions/mean_length": 206.2,
|
|
"completions/mean_terminated_length": 206.2,
|
|
"completions/min_length": 147.2,
|
|
"completions/min_terminated_length": 147.2,
|
|
"entropy": 1.0438854165375233,
|
|
"epoch": 0.10756642456559713,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.24372972548007965,
|
|
"kl": 0.010773162270197645,
|
|
"learning_rate": 3.581495098039216e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 5095195.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.5114813864231109,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.5114813983440399,
|
|
"sampling/importance_sampling_ratio/max": 1.2707341670989991,
|
|
"sampling/importance_sampling_ratio/mean": 1.000458264350891,
|
|
"sampling/importance_sampling_ratio/min": 0.7152528584003448,
|
|
"sampling/sampling_logp_difference/max": 0.35886303782463075,
|
|
"sampling/sampling_logp_difference/mean": 0.022765771113336088,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.8,
|
|
"completions/max_terminated_length": 271.8,
|
|
"completions/mean_length": 188.375,
|
|
"completions/mean_terminated_length": 188.375,
|
|
"completions/min_length": 123.0,
|
|
"completions/min_terminated_length": 123.0,
|
|
"entropy": 1.0524725951254368,
|
|
"epoch": 0.10848579571573044,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.10499633848667145,
|
|
"kl": 0.0076308351097395645,
|
|
"learning_rate": 3.6121323529411766e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 5138401.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.44819211661815644,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.4481921285390854,
|
|
"sampling/importance_sampling_ratio/max": 1.2385743021965028,
|
|
"sampling/importance_sampling_ratio/mean": 1.000013965368271,
|
|
"sampling/importance_sampling_ratio/min": 0.7369125068187714,
|
|
"sampling/sampling_logp_difference/max": 0.31007453203201296,
|
|
"sampling/sampling_logp_difference/mean": 0.022578140906989574,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 286.9,
|
|
"completions/max_terminated_length": 268.9,
|
|
"completions/mean_length": 190.5625,
|
|
"completions/mean_terminated_length": 187.53035888671874,
|
|
"completions/min_length": 122.4,
|
|
"completions/min_terminated_length": 122.4,
|
|
"entropy": 0.9723918177187443,
|
|
"epoch": 0.10940516686586375,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.22342681884765625,
|
|
"kl": 0.00678293082601158,
|
|
"learning_rate": 3.6427696078431375e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 5181374.0,
|
|
"reward": 0.44375,
|
|
"reward_std": 0.2882675379514694,
|
|
"rewards/reward_fn/mean": 0.44375,
|
|
"rewards/reward_fn/std": 0.2882675439119339,
|
|
"sampling/importance_sampling_ratio/max": 1.3537864804267883,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001680910587312,
|
|
"sampling/importance_sampling_ratio/min": 0.7728900909423828,
|
|
"sampling/sampling_logp_difference/max": 0.32552298307418825,
|
|
"sampling/sampling_logp_difference/mean": 0.020928254909813405,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.4,
|
|
"completions/max_terminated_length": 298.4,
|
|
"completions/mean_length": 218.75,
|
|
"completions/mean_terminated_length": 218.75,
|
|
"completions/min_length": 140.6,
|
|
"completions/min_terminated_length": 140.6,
|
|
"entropy": 1.1200744204223156,
|
|
"epoch": 0.11032453801599706,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.122947692871094,
|
|
"kl": 0.00601845326309558,
|
|
"learning_rate": 3.6734068627450984e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 5228362.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.5417128652334213,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.5417128771543502,
|
|
"sampling/importance_sampling_ratio/max": 1.304845130443573,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999196469783783,
|
|
"sampling/importance_sampling_ratio/min": 0.7648558259010315,
|
|
"sampling/sampling_logp_difference/max": 0.29331214427948,
|
|
"sampling/sampling_logp_difference/mean": 0.022312159277498723,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.11032453801599706,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 298.37696335078533,
|
|
"eval_completions/max_terminated_length": 296.7312390924956,
|
|
"eval_completions/mean_length": 210.15903141361255,
|
|
"eval_completions/mean_terminated_length": 209.85418237815978,
|
|
"eval_completions/min_length": 140.44851657940663,
|
|
"eval_completions/min_terminated_length": 140.44851657940663,
|
|
"eval_entropy": 1.068753021549804,
|
|
"eval_frac_reward_zero_std": 0.33158813263525305,
|
|
"eval_kl": 0.007995697537016117,
|
|
"eval_loss": 0.0004092247982043773,
|
|
"eval_num_tokens": 5228362.0,
|
|
"eval_reward": -0.04712041884816754,
|
|
"eval_reward_std": 0.4636780871614736,
|
|
"eval_rewards/reward_fn/mean": -0.04712041884816754,
|
|
"eval_rewards/reward_fn/std": 0.4636781009964086,
|
|
"eval_runtime": 1739.5189,
|
|
"eval_samples_per_second": 0.329,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2854634885388518,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000063097705807,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7597562024990717,
|
|
"eval_sampling/sampling_logp_difference/max": 0.3159886962664273,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021668341415519803,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.9,
|
|
"completions/max_terminated_length": 324.9,
|
|
"completions/mean_length": 224.55,
|
|
"completions/mean_terminated_length": 224.55,
|
|
"completions/min_length": 147.5,
|
|
"completions/min_terminated_length": 147.5,
|
|
"entropy": 1.0434857800602912,
|
|
"epoch": 0.11124390916613036,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1081586703658104,
|
|
"kl": 0.006170208979165181,
|
|
"learning_rate": 3.7040441176470593e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 5274734.0,
|
|
"reward": -0.2375,
|
|
"reward_std": 0.40021162629127505,
|
|
"rewards/reward_fn/mean": -0.2375,
|
|
"rewards/reward_fn/std": 0.4002116322517395,
|
|
"sampling/importance_sampling_ratio/max": 1.2809426426887511,
|
|
"sampling/importance_sampling_ratio/mean": 0.999780023097992,
|
|
"sampling/importance_sampling_ratio/min": 0.7142409771680832,
|
|
"sampling/sampling_logp_difference/max": 0.44220882654190063,
|
|
"sampling/sampling_logp_difference/mean": 0.021672898903489113,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.3,
|
|
"completions/max_terminated_length": 289.3,
|
|
"completions/mean_length": 210.675,
|
|
"completions/mean_terminated_length": 210.675,
|
|
"completions/min_length": 134.5,
|
|
"completions/min_terminated_length": 134.5,
|
|
"entropy": 0.968850920163095,
|
|
"epoch": 0.11216328031626367,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.864732265472412,
|
|
"kl": 0.009097536602712352,
|
|
"learning_rate": 3.73468137254902e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 5316476.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.5061949193477631,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.506194931268692,
|
|
"sampling/importance_sampling_ratio/max": 1.3063884854316712,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998998701572418,
|
|
"sampling/importance_sampling_ratio/min": 0.737203985452652,
|
|
"sampling/sampling_logp_difference/max": 0.3477449119091034,
|
|
"sampling/sampling_logp_difference/mean": 0.02080409824848175,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 316.6,
|
|
"completions/max_terminated_length": 298.8,
|
|
"completions/mean_length": 222.725,
|
|
"completions/mean_terminated_length": 219.13750152587892,
|
|
"completions/min_length": 140.8,
|
|
"completions/min_terminated_length": 140.8,
|
|
"entropy": 1.1194646969437598,
|
|
"epoch": 0.11308265146639698,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.690155506134033,
|
|
"kl": 0.011455826633027754,
|
|
"learning_rate": 3.7653186274509808e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 5358758.0,
|
|
"reward": -0.6,
|
|
"reward_std": 0.5313313931226731,
|
|
"rewards/reward_fn/mean": -0.6,
|
|
"rewards/reward_fn/std": 0.5313313931226731,
|
|
"sampling/importance_sampling_ratio/max": 1.306610071659088,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002062559127807,
|
|
"sampling/importance_sampling_ratio/min": 0.7746486663818359,
|
|
"sampling/sampling_logp_difference/max": 0.2955535888671875,
|
|
"sampling/sampling_logp_difference/mean": 0.021349863335490226,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.3,
|
|
"completions/max_terminated_length": 300.3,
|
|
"completions/mean_length": 228.9875,
|
|
"completions/mean_terminated_length": 228.9875,
|
|
"completions/min_length": 158.6,
|
|
"completions/min_terminated_length": 158.6,
|
|
"entropy": 1.1286462068557739,
|
|
"epoch": 0.1140020226165303,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.212457656860352,
|
|
"kl": 0.00790575742139481,
|
|
"learning_rate": 3.7959558823529413e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 5405533.0,
|
|
"reward": 0.23125,
|
|
"reward_std": 0.577428537607193,
|
|
"rewards/reward_fn/mean": 0.23125,
|
|
"rewards/reward_fn/std": 0.5774285733699799,
|
|
"sampling/importance_sampling_ratio/max": 1.2457308173179626,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003167986869812,
|
|
"sampling/importance_sampling_ratio/min": 0.8062489449977874,
|
|
"sampling/sampling_logp_difference/max": 0.23042677640914916,
|
|
"sampling/sampling_logp_difference/mean": 0.021200483851134776,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 265.0,
|
|
"completions/max_terminated_length": 265.0,
|
|
"completions/mean_length": 203.4,
|
|
"completions/mean_terminated_length": 203.4,
|
|
"completions/min_length": 146.4,
|
|
"completions/min_terminated_length": 146.4,
|
|
"entropy": 1.0522544912993907,
|
|
"epoch": 0.1149213937666636,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.13615116477012634,
|
|
"kl": 0.009838560479693115,
|
|
"learning_rate": 3.826593137254902e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 5446733.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.6745196104049682,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.6745196282863617,
|
|
"sampling/importance_sampling_ratio/max": 1.283636200428009,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004195511341094,
|
|
"sampling/importance_sampling_ratio/min": 0.734540593624115,
|
|
"sampling/sampling_logp_difference/max": 0.3464024543762207,
|
|
"sampling/sampling_logp_difference/mean": 0.021544823609292506,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.3,
|
|
"completions/max_terminated_length": 283.3,
|
|
"completions/mean_length": 202.3875,
|
|
"completions/mean_terminated_length": 202.3875,
|
|
"completions/min_length": 135.0,
|
|
"completions/min_terminated_length": 135.0,
|
|
"entropy": 1.0066056750714778,
|
|
"epoch": 0.11584076491679692,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.153637409210205,
|
|
"kl": 0.008987490230356342,
|
|
"learning_rate": 3.857230392156863e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 5484028.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.4570765882730484,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.45707660615444184,
|
|
"sampling/importance_sampling_ratio/max": 1.281479012966156,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997131764888764,
|
|
"sampling/importance_sampling_ratio/min": 0.7385365486145019,
|
|
"sampling/sampling_logp_difference/max": 0.3457703173160553,
|
|
"sampling/sampling_logp_difference/mean": 0.020664984360337257,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.1,
|
|
"completions/max_terminated_length": 305.1,
|
|
"completions/mean_length": 217.125,
|
|
"completions/mean_terminated_length": 217.125,
|
|
"completions/min_length": 140.9,
|
|
"completions/min_terminated_length": 140.9,
|
|
"entropy": 1.1688122861087322,
|
|
"epoch": 0.11676013606693023,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.08216671645641327,
|
|
"kl": 0.006927935587009415,
|
|
"learning_rate": 3.887867647058824e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 5528502.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.32558145225048063,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.3255814641714096,
|
|
"sampling/importance_sampling_ratio/max": 1.278193998336792,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003093004226684,
|
|
"sampling/importance_sampling_ratio/min": 0.7788882911205292,
|
|
"sampling/sampling_logp_difference/max": 0.27155337035655974,
|
|
"sampling/sampling_logp_difference/mean": 0.023232090286910534,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 289.4,
|
|
"completions/max_terminated_length": 271.4,
|
|
"completions/mean_length": 193.1875,
|
|
"completions/mean_terminated_length": 189.93035888671875,
|
|
"completions/min_length": 120.2,
|
|
"completions/min_terminated_length": 120.2,
|
|
"entropy": 1.0665395490825176,
|
|
"epoch": 0.11767950721706352,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.827215194702148,
|
|
"kl": 0.00874875822046306,
|
|
"learning_rate": 3.918504901960785e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 5571653.0,
|
|
"reward": -0.03125,
|
|
"reward_std": 0.2973824992775917,
|
|
"rewards/reward_fn/mean": -0.03125,
|
|
"rewards/reward_fn/std": 0.2973825052380562,
|
|
"sampling/importance_sampling_ratio/max": 1.3046435356140136,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001174807548523,
|
|
"sampling/importance_sampling_ratio/min": 0.7995468974113464,
|
|
"sampling/sampling_logp_difference/max": 0.2812245011329651,
|
|
"sampling/sampling_logp_difference/mean": 0.021241790615022182,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 246.9,
|
|
"completions/max_terminated_length": 246.9,
|
|
"completions/mean_length": 171.55,
|
|
"completions/mean_terminated_length": 171.55,
|
|
"completions/min_length": 115.2,
|
|
"completions/min_terminated_length": 115.2,
|
|
"entropy": 0.9502380028367042,
|
|
"epoch": 0.11859887836719683,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.376691818237305,
|
|
"kl": 0.012134195037651807,
|
|
"learning_rate": 3.9491421568627455e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 5607961.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.39840384423732755,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.39840384423732755,
|
|
"sampling/importance_sampling_ratio/max": 1.3476706624031067,
|
|
"sampling/importance_sampling_ratio/mean": 1.0007337510585785,
|
|
"sampling/importance_sampling_ratio/min": 0.7455664277076721,
|
|
"sampling/sampling_logp_difference/max": 0.36572521924972534,
|
|
"sampling/sampling_logp_difference/mean": 0.021275184117257595,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.0,
|
|
"completions/max_terminated_length": 288.0,
|
|
"completions/mean_length": 197.75,
|
|
"completions/mean_terminated_length": 197.75,
|
|
"completions/min_length": 141.5,
|
|
"completions/min_terminated_length": 141.5,
|
|
"entropy": 1.0110009878873825,
|
|
"epoch": 0.11951824951733014,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.256997108459473,
|
|
"kl": 0.009620112203992903,
|
|
"learning_rate": 3.979779411764706e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 5651389.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.370937442779541,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.370937442779541,
|
|
"sampling/importance_sampling_ratio/max": 1.2458324909210206,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999366641044617,
|
|
"sampling/importance_sampling_ratio/min": 0.7940617024898529,
|
|
"sampling/sampling_logp_difference/max": 0.23945897817611694,
|
|
"sampling/sampling_logp_difference/mean": 0.020540885999798773,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.9,
|
|
"completions/max_terminated_length": 291.9,
|
|
"completions/mean_length": 215.7125,
|
|
"completions/mean_terminated_length": 215.7125,
|
|
"completions/min_length": 144.6,
|
|
"completions/min_terminated_length": 144.6,
|
|
"entropy": 1.045546467602253,
|
|
"epoch": 0.12043762066746345,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.2061614990234375,
|
|
"kl": 0.00735490906808991,
|
|
"learning_rate": 4.010416666666667e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 5696502.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.6018513649702072,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.6018513709306716,
|
|
"sampling/importance_sampling_ratio/max": 1.2397029757499696,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003082811832429,
|
|
"sampling/importance_sampling_ratio/min": 0.7763580977916718,
|
|
"sampling/sampling_logp_difference/max": 0.2685515284538269,
|
|
"sampling/sampling_logp_difference/mean": 0.021022022888064384,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 307.6,
|
|
"completions/max_terminated_length": 294.9,
|
|
"completions/mean_length": 231.25,
|
|
"completions/mean_terminated_length": 226.32916870117188,
|
|
"completions/min_length": 160.6,
|
|
"completions/min_terminated_length": 160.6,
|
|
"entropy": 1.1383093502372503,
|
|
"epoch": 0.12135699181759677,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.147118091583252,
|
|
"kl": 0.00999721254920587,
|
|
"learning_rate": 4.041053921568628e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 5740474.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.4475270092487335,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.44752702713012693,
|
|
"sampling/importance_sampling_ratio/max": 1.2792275428771973,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997247695922852,
|
|
"sampling/importance_sampling_ratio/min": 0.7661856114864349,
|
|
"sampling/sampling_logp_difference/max": 0.3035975515842438,
|
|
"sampling/sampling_logp_difference/mean": 0.022428595181554555,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 338.4,
|
|
"completions/max_terminated_length": 324.1,
|
|
"completions/mean_length": 241.2,
|
|
"completions/mean_terminated_length": 238.375,
|
|
"completions/min_length": 168.0,
|
|
"completions/min_terminated_length": 168.0,
|
|
"entropy": 1.043086964637041,
|
|
"epoch": 0.12227636296773008,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.07285313308238983,
|
|
"kl": 0.009247479937039315,
|
|
"learning_rate": 4.071691176470588e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 5788738.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.5789267063140869,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.5789267361164093,
|
|
"sampling/importance_sampling_ratio/max": 1.2721714496612548,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001012861728669,
|
|
"sampling/importance_sampling_ratio/min": 0.7748051226139069,
|
|
"sampling/sampling_logp_difference/max": 0.27231634259223936,
|
|
"sampling/sampling_logp_difference/mean": 0.020653835125267507,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 317.4,
|
|
"completions/max_terminated_length": 317.4,
|
|
"completions/mean_length": 210.8375,
|
|
"completions/mean_terminated_length": 210.8375,
|
|
"completions/min_length": 138.5,
|
|
"completions/min_terminated_length": 138.5,
|
|
"entropy": 1.041532776877284,
|
|
"epoch": 0.12319573411786339,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.706281661987305,
|
|
"kl": 0.01274330764717888,
|
|
"learning_rate": 4.102328431372549e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 5833565.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.5324686735868454,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.5324686974287033,
|
|
"sampling/importance_sampling_ratio/max": 1.2627211689949036,
|
|
"sampling/importance_sampling_ratio/mean": 1.000657558441162,
|
|
"sampling/importance_sampling_ratio/min": 0.767503845691681,
|
|
"sampling/sampling_logp_difference/max": 0.2748436689376831,
|
|
"sampling/sampling_logp_difference/mean": 0.021594073809683322,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.6,
|
|
"completions/max_terminated_length": 283.6,
|
|
"completions/mean_length": 199.45,
|
|
"completions/mean_terminated_length": 199.45,
|
|
"completions/min_length": 133.7,
|
|
"completions/min_terminated_length": 133.7,
|
|
"entropy": 1.0785112269222736,
|
|
"epoch": 0.12411510526799668,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.028674125671387,
|
|
"kl": 0.010370943482848815,
|
|
"learning_rate": 4.13296568627451e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 5878649.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.4265191316604614,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.42651914358139037,
|
|
"sampling/importance_sampling_ratio/max": 1.2595079779624938,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000385701656342,
|
|
"sampling/importance_sampling_ratio/min": 0.7906409800052643,
|
|
"sampling/sampling_logp_difference/max": 0.251240661740303,
|
|
"sampling/sampling_logp_difference/mean": 0.021430917829275132,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.0,
|
|
"completions/max_terminated_length": 302.0,
|
|
"completions/mean_length": 206.3625,
|
|
"completions/mean_terminated_length": 206.3625,
|
|
"completions/min_length": 126.9,
|
|
"completions/min_terminated_length": 126.9,
|
|
"entropy": 1.0548729564994574,
|
|
"epoch": 0.12503447641813,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.1075732409954071,
|
|
"kl": 0.011580927346949465,
|
|
"learning_rate": 4.1636029411764715e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 5926526.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.7203713327646255,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.7203713744878769,
|
|
"sampling/importance_sampling_ratio/max": 1.3204351902008056,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999918341636658,
|
|
"sampling/importance_sampling_ratio/min": 0.7800904512405396,
|
|
"sampling/sampling_logp_difference/max": 0.3146080493927002,
|
|
"sampling/sampling_logp_difference/mean": 0.02158144675195217,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 264.9,
|
|
"completions/max_terminated_length": 264.9,
|
|
"completions/mean_length": 187.75,
|
|
"completions/mean_terminated_length": 187.75,
|
|
"completions/min_length": 113.1,
|
|
"completions/min_terminated_length": 113.1,
|
|
"entropy": 1.035442354902625,
|
|
"epoch": 0.1259538475682633,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.101892471313477,
|
|
"kl": 0.016452591243432835,
|
|
"learning_rate": 4.194240196078432e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 5966618.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.4372564494609833,
|
|
"rewards/reward_fn/mean": 0.375,
|
|
"rewards/reward_fn/std": 0.4372564613819122,
|
|
"sampling/importance_sampling_ratio/max": 1.2982187032699586,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996362388134002,
|
|
"sampling/importance_sampling_ratio/min": 0.6908780992031097,
|
|
"sampling/sampling_logp_difference/max": 0.4562541127204895,
|
|
"sampling/sampling_logp_difference/mean": 0.021488183457404376,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 336.7,
|
|
"completions/max_terminated_length": 336.7,
|
|
"completions/mean_length": 223.9875,
|
|
"completions/mean_terminated_length": 223.9875,
|
|
"completions/min_length": 148.3,
|
|
"completions/min_terminated_length": 148.3,
|
|
"entropy": 1.1679816335439681,
|
|
"epoch": 0.12687321871839663,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.767652988433838,
|
|
"kl": 0.012456041463883594,
|
|
"learning_rate": 4.2248774509803925e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 6011585.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.5558438330888749,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.5558438569307327,
|
|
"sampling/importance_sampling_ratio/max": 1.334365463256836,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999495685100556,
|
|
"sampling/importance_sampling_ratio/min": 0.7871855437755585,
|
|
"sampling/sampling_logp_difference/max": 0.36275808811187743,
|
|
"sampling/sampling_logp_difference/mean": 0.022078605182468892,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 338.8,
|
|
"completions/max_terminated_length": 338.8,
|
|
"completions/mean_length": 242.225,
|
|
"completions/mean_terminated_length": 242.225,
|
|
"completions/min_length": 173.9,
|
|
"completions/min_terminated_length": 173.9,
|
|
"entropy": 1.1290132313966752,
|
|
"epoch": 0.12779258986852993,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.37898588180542,
|
|
"kl": 0.02128637230489403,
|
|
"learning_rate": 4.255514705882353e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 6060027.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.45775099098682404,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.4577509969472885,
|
|
"sampling/importance_sampling_ratio/max": 1.3249101161956787,
|
|
"sampling/importance_sampling_ratio/mean": 1.000572180747986,
|
|
"sampling/importance_sampling_ratio/min": 0.7810700714588166,
|
|
"sampling/sampling_logp_difference/max": 0.3053713947534561,
|
|
"sampling/sampling_logp_difference/mean": 0.021317550167441367,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 327.1,
|
|
"completions/max_terminated_length": 327.1,
|
|
"completions/mean_length": 236.4875,
|
|
"completions/mean_terminated_length": 236.4875,
|
|
"completions/min_length": 161.9,
|
|
"completions/min_terminated_length": 161.9,
|
|
"entropy": 1.135564436763525,
|
|
"epoch": 0.12871196101866322,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.217648983001709,
|
|
"kl": 0.024721204745583238,
|
|
"learning_rate": 4.2861519607843135e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 6107578.0,
|
|
"reward": -0.225,
|
|
"reward_std": 0.4943925529718399,
|
|
"rewards/reward_fn/mean": -0.225,
|
|
"rewards/reward_fn/std": 0.4943925827741623,
|
|
"sampling/importance_sampling_ratio/max": 1.3563641548156737,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998765587806702,
|
|
"sampling/importance_sampling_ratio/min": 0.7215391218662262,
|
|
"sampling/sampling_logp_difference/max": 0.34402431845664977,
|
|
"sampling/sampling_logp_difference/mean": 0.021769707649946214,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.12871196101866322,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0030541012216404886,
|
|
"eval_completions/max_length": 321.89877835951137,
|
|
"eval_completions/max_terminated_length": 319.60034904013963,
|
|
"eval_completions/mean_length": 225.65728621291447,
|
|
"eval_completions/mean_terminated_length": 224.96231220238704,
|
|
"eval_completions/min_length": 150.32460732984293,
|
|
"eval_completions/min_terminated_length": 150.32460732984293,
|
|
"eval_entropy": 1.1268045640116586,
|
|
"eval_frac_reward_zero_std": 0.2530541012216405,
|
|
"eval_kl": 0.019346026554996273,
|
|
"eval_loss": 0.0010174785275012255,
|
|
"eval_num_tokens": 6107578.0,
|
|
"eval_reward": -0.027923211169284468,
|
|
"eval_reward_std": 0.45558283786187,
|
|
"eval_rewards/reward_fn/mean": -0.027923211169284468,
|
|
"eval_rewards/reward_fn/std": 0.4555828505525622,
|
|
"eval_runtime": 1859.0437,
|
|
"eval_samples_per_second": 0.308,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.298839797108169,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999097204957333,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7600866199241882,
|
|
"eval_sampling/sampling_logp_difference/max": 0.3255701512878478,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02160237773271183,
|
|
"eval_steps_per_second": 0.039,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.2,
|
|
"completions/max_terminated_length": 284.2,
|
|
"completions/mean_length": 210.75,
|
|
"completions/mean_terminated_length": 210.75,
|
|
"completions/min_length": 143.6,
|
|
"completions/min_terminated_length": 143.6,
|
|
"entropy": 1.092452796176076,
|
|
"epoch": 0.12963133216879655,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.544597625732422,
|
|
"kl": 0.014508970058523118,
|
|
"learning_rate": 4.316789215686275e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 6151286.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.4303868621587753,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.4303868681192398,
|
|
"sampling/importance_sampling_ratio/max": 1.2749321103096007,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000078678131104,
|
|
"sampling/importance_sampling_ratio/min": 0.7807251989841462,
|
|
"sampling/sampling_logp_difference/max": 0.2800210028886795,
|
|
"sampling/sampling_logp_difference/mean": 0.021823861822485923,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.9,
|
|
"completions/max_terminated_length": 283.9,
|
|
"completions/mean_length": 217.1125,
|
|
"completions/mean_terminated_length": 217.1125,
|
|
"completions/min_length": 142.6,
|
|
"completions/min_terminated_length": 142.6,
|
|
"entropy": 1.101941668987274,
|
|
"epoch": 0.13055070331892984,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.590672969818115,
|
|
"kl": 0.013525198728893883,
|
|
"learning_rate": 4.347426470588235e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 6194975.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.34659049212932586,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.34659049212932586,
|
|
"sampling/importance_sampling_ratio/max": 1.2486390590667724,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002484798431397,
|
|
"sampling/importance_sampling_ratio/min": 0.73974589407444,
|
|
"sampling/sampling_logp_difference/max": 0.31851269006729127,
|
|
"sampling/sampling_logp_difference/mean": 0.021292382664978505,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 328.3,
|
|
"completions/max_terminated_length": 328.3,
|
|
"completions/mean_length": 226.0625,
|
|
"completions/mean_terminated_length": 226.0625,
|
|
"completions/min_length": 141.5,
|
|
"completions/min_terminated_length": 141.5,
|
|
"entropy": 1.0521254802122713,
|
|
"epoch": 0.13147007446906317,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.299581527709961,
|
|
"kl": 0.025897987411008218,
|
|
"learning_rate": 4.378063725490197e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 6237108.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.6083779186010361,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.6083779543638229,
|
|
"sampling/importance_sampling_ratio/max": 1.3657152652740479,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999037444591522,
|
|
"sampling/importance_sampling_ratio/min": 0.7124147206544876,
|
|
"sampling/sampling_logp_difference/max": 0.41489108800888064,
|
|
"sampling/sampling_logp_difference/mean": 0.021135816909372808,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 359.9,
|
|
"completions/max_terminated_length": 328.8,
|
|
"completions/mean_length": 244.0625,
|
|
"completions/mean_terminated_length": 238.15535888671874,
|
|
"completions/min_length": 155.9,
|
|
"completions/min_terminated_length": 155.9,
|
|
"entropy": 1.1280117265880107,
|
|
"epoch": 0.13238944561919647,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.672699928283691,
|
|
"kl": 0.02738113821251318,
|
|
"learning_rate": 4.408700980392157e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 6281409.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.6525094747543335,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.6525094985961915,
|
|
"sampling/importance_sampling_ratio/max": 1.3170461773872375,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996939182281495,
|
|
"sampling/importance_sampling_ratio/min": 0.7692372381687165,
|
|
"sampling/sampling_logp_difference/max": 0.315592485666275,
|
|
"sampling/sampling_logp_difference/mean": 0.02136859782040119,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.2,
|
|
"completions/max_terminated_length": 306.2,
|
|
"completions/mean_length": 206.0,
|
|
"completions/mean_terminated_length": 206.0,
|
|
"completions/min_length": 139.3,
|
|
"completions/min_terminated_length": 139.3,
|
|
"entropy": 1.0977587856352329,
|
|
"epoch": 0.1333088167693298,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.023177623748779,
|
|
"kl": 0.015292615463840776,
|
|
"learning_rate": 4.439338235294118e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 6323561.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.42727080881595614,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.42727082669734956,
|
|
"sampling/importance_sampling_ratio/max": 1.3277016520500182,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003354728221894,
|
|
"sampling/importance_sampling_ratio/min": 0.7874673068523407,
|
|
"sampling/sampling_logp_difference/max": 0.30552958250045775,
|
|
"sampling/sampling_logp_difference/mean": 0.021854222379624845,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 327.2,
|
|
"completions/max_terminated_length": 327.2,
|
|
"completions/mean_length": 229.3,
|
|
"completions/mean_terminated_length": 229.3,
|
|
"completions/min_length": 148.5,
|
|
"completions/min_terminated_length": 148.5,
|
|
"entropy": 1.0221133209764957,
|
|
"epoch": 0.1342281879194631,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.08343011140823364,
|
|
"kl": 0.017870039975969122,
|
|
"learning_rate": 4.469975490196079e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 6369817.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.5809791445732116,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.580979174375534,
|
|
"sampling/importance_sampling_ratio/max": 1.2901687741279602,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001559674739837,
|
|
"sampling/importance_sampling_ratio/min": 0.7431910753250122,
|
|
"sampling/sampling_logp_difference/max": 0.3421707391738892,
|
|
"sampling/sampling_logp_difference/mean": 0.020285730436444282,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.3,
|
|
"completions/max_terminated_length": 275.3,
|
|
"completions/mean_length": 201.275,
|
|
"completions/mean_terminated_length": 201.275,
|
|
"completions/min_length": 135.0,
|
|
"completions/min_terminated_length": 135.0,
|
|
"entropy": 1.0342267520725728,
|
|
"epoch": 0.13514755906959638,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.92233943939209,
|
|
"kl": 0.01684942019637674,
|
|
"learning_rate": 4.5006127450980395e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 6410055.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.3355525702238083,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.3355525702238083,
|
|
"sampling/importance_sampling_ratio/max": 1.2795489192008973,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997927188873291,
|
|
"sampling/importance_sampling_ratio/min": 0.7806328356266021,
|
|
"sampling/sampling_logp_difference/max": 0.28694812059402464,
|
|
"sampling/sampling_logp_difference/mean": 0.020654335059225558,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.0,
|
|
"completions/max_terminated_length": 269.0,
|
|
"completions/mean_length": 199.2375,
|
|
"completions/mean_terminated_length": 199.2375,
|
|
"completions/min_length": 137.1,
|
|
"completions/min_terminated_length": 137.1,
|
|
"entropy": 0.9882201224565506,
|
|
"epoch": 0.1360669302197297,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 8.399575233459473,
|
|
"kl": 0.026593866568873636,
|
|
"learning_rate": 4.53125e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 6452914.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.4142331928014755,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.41423319876194,
|
|
"sampling/importance_sampling_ratio/max": 1.2976352095603942,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002907395362854,
|
|
"sampling/importance_sampling_ratio/min": 0.7600722372531891,
|
|
"sampling/sampling_logp_difference/max": 0.32167547941207886,
|
|
"sampling/sampling_logp_difference/mean": 0.020902066119015218,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.0,
|
|
"completions/max_terminated_length": 289.0,
|
|
"completions/mean_length": 203.225,
|
|
"completions/mean_terminated_length": 203.225,
|
|
"completions/min_length": 142.2,
|
|
"completions/min_terminated_length": 142.2,
|
|
"entropy": 1.0744589954614638,
|
|
"epoch": 0.136986301369863,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 8.661561012268066,
|
|
"kl": 0.03353368609678,
|
|
"learning_rate": 4.561887254901961e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 6495732.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.5512929946184159,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.5512930184602738,
|
|
"sampling/importance_sampling_ratio/max": 1.3335409760475159,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998663485050201,
|
|
"sampling/importance_sampling_ratio/min": 0.7325195729732513,
|
|
"sampling/sampling_logp_difference/max": 0.4317288875579834,
|
|
"sampling/sampling_logp_difference/mean": 0.021174842864274977,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.4,
|
|
"completions/max_terminated_length": 281.4,
|
|
"completions/mean_length": 199.4875,
|
|
"completions/mean_terminated_length": 199.4875,
|
|
"completions/min_length": 134.2,
|
|
"completions/min_terminated_length": 134.2,
|
|
"entropy": 1.0627880647778511,
|
|
"epoch": 0.13790567251999633,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.16991867125034332,
|
|
"kl": 0.022514234139816836,
|
|
"learning_rate": 4.592524509803922e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 6542563.0,
|
|
"reward": -0.2875,
|
|
"reward_std": 0.41911540925502777,
|
|
"rewards/reward_fn/mean": -0.2875,
|
|
"rewards/reward_fn/std": 0.4191154152154922,
|
|
"sampling/importance_sampling_ratio/max": 1.3318233966827393,
|
|
"sampling/importance_sampling_ratio/mean": 1.000492697954178,
|
|
"sampling/importance_sampling_ratio/min": 0.755594152212143,
|
|
"sampling/sampling_logp_difference/max": 0.3407777100801468,
|
|
"sampling/sampling_logp_difference/mean": 0.02217220440506935,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 253.8,
|
|
"completions/max_terminated_length": 253.8,
|
|
"completions/mean_length": 181.0125,
|
|
"completions/mean_terminated_length": 181.0125,
|
|
"completions/min_length": 123.2,
|
|
"completions/min_terminated_length": 123.2,
|
|
"entropy": 1.0705828636884689,
|
|
"epoch": 0.13882504367012963,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.78505802154541,
|
|
"kl": 0.029351890063844623,
|
|
"learning_rate": 4.623161764705883e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 6587300.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.44087620079517365,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.44087620079517365,
|
|
"sampling/importance_sampling_ratio/max": 1.3288820385932922,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002204418182372,
|
|
"sampling/importance_sampling_ratio/min": 0.7853626430034637,
|
|
"sampling/sampling_logp_difference/max": 0.3062667608261108,
|
|
"sampling/sampling_logp_difference/mean": 0.02165432758629322,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 237.7,
|
|
"completions/max_terminated_length": 237.7,
|
|
"completions/mean_length": 182.9375,
|
|
"completions/mean_terminated_length": 182.9375,
|
|
"completions/min_length": 131.5,
|
|
"completions/min_terminated_length": 131.5,
|
|
"entropy": 1.050403956323862,
|
|
"epoch": 0.13974441482026295,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.823770999908447,
|
|
"kl": 0.032556075253523886,
|
|
"learning_rate": 4.653799019607844e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 6629303.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.4424272894859314,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.44242729544639586,
|
|
"sampling/importance_sampling_ratio/max": 1.4264759182929994,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998594403266907,
|
|
"sampling/importance_sampling_ratio/min": 0.7270286232233047,
|
|
"sampling/sampling_logp_difference/max": 0.3886754333972931,
|
|
"sampling/sampling_logp_difference/mean": 0.021766609512269497,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.2,
|
|
"completions/max_terminated_length": 284.2,
|
|
"completions/mean_length": 203.5875,
|
|
"completions/mean_terminated_length": 203.5875,
|
|
"completions/min_length": 126.8,
|
|
"completions/min_terminated_length": 126.8,
|
|
"entropy": 1.0652003578841687,
|
|
"epoch": 0.14066378597039625,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 15.634310722351074,
|
|
"kl": 0.027967221062863246,
|
|
"learning_rate": 4.684436274509804e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 6672702.0,
|
|
"reward": -0.2375,
|
|
"reward_std": 0.3518119215965271,
|
|
"rewards/reward_fn/mean": -0.2375,
|
|
"rewards/reward_fn/std": 0.3518119215965271,
|
|
"sampling/importance_sampling_ratio/max": 1.2514676809310914,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001472532749176,
|
|
"sampling/importance_sampling_ratio/min": 0.8009488821029663,
|
|
"sampling/sampling_logp_difference/max": 0.23702155351638793,
|
|
"sampling/sampling_logp_difference/mean": 0.021141395904123783,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 329.9,
|
|
"completions/max_terminated_length": 324.9,
|
|
"completions/mean_length": 228.2875,
|
|
"completions/mean_terminated_length": 225.59285888671874,
|
|
"completions/min_length": 149.8,
|
|
"completions/min_terminated_length": 149.8,
|
|
"entropy": 1.1336680881679058,
|
|
"epoch": 0.14158315712052955,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.894134998321533,
|
|
"kl": 0.0194727988622617,
|
|
"learning_rate": 4.715073529411765e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 6721093.0,
|
|
"reward": 0.20625,
|
|
"reward_std": 0.6140542179346085,
|
|
"rewards/reward_fn/mean": 0.20625,
|
|
"rewards/reward_fn/std": 0.6140542298555374,
|
|
"sampling/importance_sampling_ratio/max": 1.303449809551239,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005681693553925,
|
|
"sampling/importance_sampling_ratio/min": 0.7986957371234894,
|
|
"sampling/sampling_logp_difference/max": 0.26659847497940065,
|
|
"sampling/sampling_logp_difference/mean": 0.022179079987108708,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.4,
|
|
"completions/max_terminated_length": 258.4,
|
|
"completions/mean_length": 190.85,
|
|
"completions/mean_terminated_length": 190.85,
|
|
"completions/min_length": 126.8,
|
|
"completions/min_terminated_length": 126.8,
|
|
"entropy": 1.069445087760687,
|
|
"epoch": 0.14250252827066287,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.343962669372559,
|
|
"kl": 0.020104502828326078,
|
|
"learning_rate": 4.745710784313726e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 6762465.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.46734622716903684,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.46734623312950135,
|
|
"sampling/importance_sampling_ratio/max": 1.3031768798828125,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997886836528778,
|
|
"sampling/importance_sampling_ratio/min": 0.7159755289554596,
|
|
"sampling/sampling_logp_difference/max": 0.35821783542633057,
|
|
"sampling/sampling_logp_difference/mean": 0.02124423012137413,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.3,
|
|
"completions/max_terminated_length": 290.3,
|
|
"completions/mean_length": 202.2875,
|
|
"completions/mean_terminated_length": 202.2875,
|
|
"completions/min_length": 144.3,
|
|
"completions/min_terminated_length": 144.3,
|
|
"entropy": 1.1350328363478184,
|
|
"epoch": 0.14342189942079617,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.09912633895874,
|
|
"kl": 0.014210601139348,
|
|
"learning_rate": 4.7763480392156866e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 6806336.0,
|
|
"reward": -0.35,
|
|
"reward_std": 0.5423088639974594,
|
|
"rewards/reward_fn/mean": -0.35,
|
|
"rewards/reward_fn/std": 0.5423088699579239,
|
|
"sampling/importance_sampling_ratio/max": 1.2767892241477967,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004692196846008,
|
|
"sampling/importance_sampling_ratio/min": 0.7695710301399231,
|
|
"sampling/sampling_logp_difference/max": 0.27397539615631106,
|
|
"sampling/sampling_logp_difference/mean": 0.02238109279423952,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 243.0,
|
|
"completions/max_terminated_length": 243.0,
|
|
"completions/mean_length": 174.2625,
|
|
"completions/mean_terminated_length": 174.2625,
|
|
"completions/min_length": 114.8,
|
|
"completions/min_terminated_length": 114.8,
|
|
"entropy": 1.1513236083090306,
|
|
"epoch": 0.1443412705709295,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.1542863845825195,
|
|
"kl": 0.015671954827848823,
|
|
"learning_rate": 4.806985294117648e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 6847109.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.5408749014139176,
|
|
"rewards/reward_fn/mean": 0.3375,
|
|
"rewards/reward_fn/std": 0.54087493121624,
|
|
"sampling/importance_sampling_ratio/max": 1.225877845287323,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997000813484191,
|
|
"sampling/importance_sampling_ratio/min": 0.7958498239517212,
|
|
"sampling/sampling_logp_difference/max": 0.2364953875541687,
|
|
"sampling/sampling_logp_difference/mean": 0.022214539349079132,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 243.6,
|
|
"completions/max_terminated_length": 243.6,
|
|
"completions/mean_length": 163.0625,
|
|
"completions/mean_terminated_length": 163.0625,
|
|
"completions/min_length": 106.9,
|
|
"completions/min_terminated_length": 106.9,
|
|
"entropy": 1.007540236413479,
|
|
"epoch": 0.1452606417210628,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.1776862144470215,
|
|
"kl": 0.024297343887155876,
|
|
"learning_rate": 4.837622549019608e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 6884130.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.42642940282821656,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.42642942070961,
|
|
"sampling/importance_sampling_ratio/max": 1.2720715641975402,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998812735080719,
|
|
"sampling/importance_sampling_ratio/min": 0.7560014605522156,
|
|
"sampling/sampling_logp_difference/max": 0.2988255798816681,
|
|
"sampling/sampling_logp_difference/mean": 0.02160265054553747,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 231.2,
|
|
"completions/max_terminated_length": 231.2,
|
|
"completions/mean_length": 165.8875,
|
|
"completions/mean_terminated_length": 165.8875,
|
|
"completions/min_length": 106.7,
|
|
"completions/min_terminated_length": 106.7,
|
|
"entropy": 1.0177243258804083,
|
|
"epoch": 0.1461800128711961,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.14829987287521362,
|
|
"kl": 0.01376922951603774,
|
|
"learning_rate": 4.868259803921569e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 6922569.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.24237500131130219,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.2423750251531601,
|
|
"sampling/importance_sampling_ratio/max": 1.235771894454956,
|
|
"sampling/importance_sampling_ratio/mean": 1.000098603963852,
|
|
"sampling/importance_sampling_ratio/min": 0.7932318806648254,
|
|
"sampling/sampling_logp_difference/max": 0.24366546869277955,
|
|
"sampling/sampling_logp_difference/mean": 0.02123898882418871,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 254.4,
|
|
"completions/max_terminated_length": 254.4,
|
|
"completions/mean_length": 190.575,
|
|
"completions/mean_terminated_length": 190.575,
|
|
"completions/min_length": 126.2,
|
|
"completions/min_terminated_length": 126.2,
|
|
"entropy": 1.1313216343522072,
|
|
"epoch": 0.1470993840213294,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.902104377746582,
|
|
"kl": 0.018426794398692437,
|
|
"learning_rate": 4.898897058823529e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 6962631.0,
|
|
"reward": -0.225,
|
|
"reward_std": 0.47630349099636077,
|
|
"rewards/reward_fn/mean": -0.225,
|
|
"rewards/reward_fn/std": 0.4763035088777542,
|
|
"sampling/importance_sampling_ratio/max": 1.3305252194404602,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998793840408325,
|
|
"sampling/importance_sampling_ratio/min": 0.7743998110294342,
|
|
"sampling/sampling_logp_difference/max": 0.3400513678789139,
|
|
"sampling/sampling_logp_difference/mean": 0.02234724797308445,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.1470993840213294,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0015270506108202443,
|
|
"eval_completions/max_length": 295.5130890052356,
|
|
"eval_completions/max_terminated_length": 294.8726003490401,
|
|
"eval_completions/mean_length": 207.77748691099475,
|
|
"eval_completions/mean_terminated_length": 207.45817768303303,
|
|
"eval_completions/min_length": 136.47993019197207,
|
|
"eval_completions/min_terminated_length": 136.47993019197207,
|
|
"eval_entropy": 1.1100842952728271,
|
|
"eval_frac_reward_zero_std": 0.2914485165794066,
|
|
"eval_kl": 0.017346000372810876,
|
|
"eval_loss": 0.0008573353989049792,
|
|
"eval_num_tokens": 6962631.0,
|
|
"eval_reward": -0.009925828970331589,
|
|
"eval_reward_std": 0.44791377201858407,
|
|
"eval_rewards/reward_fn/mean": -0.009925828970331589,
|
|
"eval_rewards/reward_fn/std": 0.4479137832529674,
|
|
"eval_runtime": 1726.0089,
|
|
"eval_samples_per_second": 0.332,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2872572171958536,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000051697927412,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7703749284174222,
|
|
"eval_sampling/sampling_logp_difference/max": 0.3057308903851434,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02126904747515501,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.6,
|
|
"completions/max_terminated_length": 319.6,
|
|
"completions/mean_length": 225.15,
|
|
"completions/mean_terminated_length": 225.15,
|
|
"completions/min_length": 159.6,
|
|
"completions/min_terminated_length": 159.6,
|
|
"entropy": 1.1622169733047485,
|
|
"epoch": 0.1480187551714627,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.971400260925293,
|
|
"kl": 0.022326471004635095,
|
|
"learning_rate": 4.929534313725491e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 7006403.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.5463842064142227,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.5463842123746872,
|
|
"sampling/importance_sampling_ratio/max": 1.3966349363327026,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003197193145752,
|
|
"sampling/importance_sampling_ratio/min": 0.7974103271961213,
|
|
"sampling/sampling_logp_difference/max": 0.3287259519100189,
|
|
"sampling/sampling_logp_difference/mean": 0.022303739003837107,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.3,
|
|
"completions/max_terminated_length": 295.3,
|
|
"completions/mean_length": 209.2625,
|
|
"completions/mean_terminated_length": 209.2625,
|
|
"completions/min_length": 141.8,
|
|
"completions/min_terminated_length": 141.8,
|
|
"entropy": 1.1289937548339366,
|
|
"epoch": 0.14893812632159603,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.15305055677890778,
|
|
"kl": 0.025693328343913892,
|
|
"learning_rate": 4.960171568627451e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 7050240.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.4124096900224686,
|
|
"rewards/reward_fn/mean": 0.09375,
|
|
"rewards/reward_fn/std": 0.41240971386432645,
|
|
"sampling/importance_sampling_ratio/max": 1.375335943698883,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000707685947419,
|
|
"sampling/importance_sampling_ratio/min": 0.6895323842763901,
|
|
"sampling/sampling_logp_difference/max": 0.4378436326980591,
|
|
"sampling/sampling_logp_difference/mean": 0.021447959914803506,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 366.7,
|
|
"completions/max_terminated_length": 361.2,
|
|
"completions/mean_length": 236.7375,
|
|
"completions/mean_terminated_length": 233.58392944335938,
|
|
"completions/min_length": 122.9,
|
|
"completions/min_terminated_length": 122.9,
|
|
"entropy": 1.1081803828477859,
|
|
"epoch": 0.14985749747172933,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.14241886138916,
|
|
"kl": 0.015311846393160522,
|
|
"learning_rate": 4.990808823529413e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 7095291.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.5733177453279495,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.5733177691698075,
|
|
"sampling/importance_sampling_ratio/max": 1.2535099864006043,
|
|
"sampling/importance_sampling_ratio/mean": 1.000777542591095,
|
|
"sampling/importance_sampling_ratio/min": 0.7887921273708344,
|
|
"sampling/sampling_logp_difference/max": 0.2520232915878296,
|
|
"sampling/sampling_logp_difference/mean": 0.021039466373622417,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.1,
|
|
"completions/max_terminated_length": 287.1,
|
|
"completions/mean_length": 196.2,
|
|
"completions/mean_terminated_length": 196.2,
|
|
"completions/min_length": 128.1,
|
|
"completions/min_terminated_length": 128.1,
|
|
"entropy": 1.1308213829994203,
|
|
"epoch": 0.15077686862186265,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.11484629660844803,
|
|
"kl": 0.019536224391777067,
|
|
"learning_rate": 4.9999993709133756e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 7135819.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.4179248958826065,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.4179249078035355,
|
|
"sampling/importance_sampling_ratio/max": 1.2677364349365234,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001580357551574,
|
|
"sampling/importance_sampling_ratio/min": 0.773081511259079,
|
|
"sampling/sampling_logp_difference/max": 0.2969019651412964,
|
|
"sampling/sampling_logp_difference/mean": 0.02234613448381424,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.5,
|
|
"completions/max_terminated_length": 272.5,
|
|
"completions/mean_length": 191.1375,
|
|
"completions/mean_terminated_length": 191.1375,
|
|
"completions/min_length": 126.1,
|
|
"completions/min_terminated_length": 126.1,
|
|
"entropy": 1.0719189211726188,
|
|
"epoch": 0.15169623977199595,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.09037373214960098,
|
|
"kl": 0.022338721586856992,
|
|
"learning_rate": 4.999996289673523e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 7174798.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.4644810765981674,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.46448108851909636,
|
|
"sampling/importance_sampling_ratio/max": 1.3307189226150513,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003674447536468,
|
|
"sampling/importance_sampling_ratio/min": 0.7516764968633651,
|
|
"sampling/sampling_logp_difference/max": 0.37522636651992797,
|
|
"sampling/sampling_logp_difference/mean": 0.02092442624270916,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.5,
|
|
"completions/max_terminated_length": 273.5,
|
|
"completions/mean_length": 196.6,
|
|
"completions/mean_terminated_length": 196.6,
|
|
"completions/min_length": 114.3,
|
|
"completions/min_terminated_length": 114.3,
|
|
"entropy": 1.0820644252002238,
|
|
"epoch": 0.15261561092212927,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.1205830574035645,
|
|
"kl": 0.018708533188328148,
|
|
"learning_rate": 4.9999906407370825e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 7216278.0,
|
|
"reward": -0.2375,
|
|
"reward_std": 0.39638522267341614,
|
|
"rewards/reward_fn/mean": -0.2375,
|
|
"rewards/reward_fn/std": 0.3963852345943451,
|
|
"sampling/importance_sampling_ratio/max": 1.4229997515678405,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998858988285064,
|
|
"sampling/importance_sampling_ratio/min": 0.758237111568451,
|
|
"sampling/sampling_logp_difference/max": 0.39238523244857787,
|
|
"sampling/sampling_logp_difference/mean": 0.02201388143002987,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 322.9,
|
|
"completions/max_terminated_length": 310.4,
|
|
"completions/mean_length": 217.075,
|
|
"completions/mean_terminated_length": 213.76250152587892,
|
|
"completions/min_length": 131.5,
|
|
"completions/min_terminated_length": 131.5,
|
|
"entropy": 1.0664160154759883,
|
|
"epoch": 0.15353498207226257,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.3463592529296875,
|
|
"kl": 0.018984398437896743,
|
|
"learning_rate": 4.999982424109855e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 7257796.0,
|
|
"reward": -0.14375,
|
|
"reward_std": 0.4889191150665283,
|
|
"rewards/reward_fn/mean": -0.14375,
|
|
"rewards/reward_fn/std": 0.4889191150665283,
|
|
"sampling/importance_sampling_ratio/max": 1.2739620685577393,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997388124465942,
|
|
"sampling/importance_sampling_ratio/min": 0.7940850377082824,
|
|
"sampling/sampling_logp_difference/max": 0.24902696609497071,
|
|
"sampling/sampling_logp_difference/mean": 0.021570471115410328,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 352.3,
|
|
"completions/max_terminated_length": 352.3,
|
|
"completions/mean_length": 236.175,
|
|
"completions/mean_terminated_length": 236.175,
|
|
"completions/min_length": 141.9,
|
|
"completions/min_terminated_length": 141.9,
|
|
"entropy": 1.0556964889168738,
|
|
"epoch": 0.15445435322239587,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.638572692871094,
|
|
"kl": 0.03635619452106766,
|
|
"learning_rate": 4.999971639800281e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 7304154.0,
|
|
"reward": 0.18125,
|
|
"reward_std": 0.45312740206718444,
|
|
"rewards/reward_fn/mean": 0.18125,
|
|
"rewards/reward_fn/std": 0.45312740802764895,
|
|
"sampling/importance_sampling_ratio/max": 1.2885913491249084,
|
|
"sampling/importance_sampling_ratio/mean": 0.999603909254074,
|
|
"sampling/importance_sampling_ratio/min": 0.7212844133377075,
|
|
"sampling/sampling_logp_difference/max": 0.36110965013504026,
|
|
"sampling/sampling_logp_difference/mean": 0.02110386434942484,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.9,
|
|
"completions/max_terminated_length": 281.9,
|
|
"completions/mean_length": 210.8,
|
|
"completions/mean_terminated_length": 210.8,
|
|
"completions/min_length": 156.5,
|
|
"completions/min_terminated_length": 156.5,
|
|
"entropy": 1.1038669049739838,
|
|
"epoch": 0.1553737243725292,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.09199784696102142,
|
|
"kl": 0.032090726867318155,
|
|
"learning_rate": 4.9999582878194345e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 7348554.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.4254381716251373,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.4254381775856018,
|
|
"sampling/importance_sampling_ratio/max": 1.2427798151969909,
|
|
"sampling/importance_sampling_ratio/mean": 1.000288063287735,
|
|
"sampling/importance_sampling_ratio/min": 0.7444603860378265,
|
|
"sampling/sampling_logp_difference/max": 0.3002180427312851,
|
|
"sampling/sampling_logp_difference/mean": 0.021241777949035166,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.6,
|
|
"completions/max_terminated_length": 284.6,
|
|
"completions/mean_length": 199.9625,
|
|
"completions/mean_terminated_length": 199.9625,
|
|
"completions/min_length": 120.5,
|
|
"completions/min_terminated_length": 120.5,
|
|
"entropy": 1.1785475850105285,
|
|
"epoch": 0.1562930955226625,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.7228684425354,
|
|
"kl": 0.04097896607709117,
|
|
"learning_rate": 4.99994236818103e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 7388871.0,
|
|
"reward": -0.14375,
|
|
"reward_std": 0.2651960164308548,
|
|
"rewards/reward_fn/mean": -0.14375,
|
|
"rewards/reward_fn/std": 0.2651960164308548,
|
|
"sampling/importance_sampling_ratio/max": 1.2273439764976501,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998808383941651,
|
|
"sampling/importance_sampling_ratio/min": 0.7310256153345108,
|
|
"sampling/sampling_logp_difference/max": 0.33525035381317136,
|
|
"sampling/sampling_logp_difference/mean": 0.022543698735535144,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.1,
|
|
"completions/max_terminated_length": 299.1,
|
|
"completions/mean_length": 210.7375,
|
|
"completions/mean_terminated_length": 210.7375,
|
|
"completions/min_length": 151.5,
|
|
"completions/min_terminated_length": 151.5,
|
|
"entropy": 1.0985402591526507,
|
|
"epoch": 0.15721246667279581,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.276334762573242,
|
|
"kl": 0.0512921616260428,
|
|
"learning_rate": 4.999923880901419e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 7435130.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.2757775604724884,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.2757775604724884,
|
|
"sampling/importance_sampling_ratio/max": 1.3340021610260009,
|
|
"sampling/importance_sampling_ratio/mean": 1.000907826423645,
|
|
"sampling/importance_sampling_ratio/min": 0.7723042488098144,
|
|
"sampling/sampling_logp_difference/max": 0.31889861822128296,
|
|
"sampling/sampling_logp_difference/mean": 0.021154037117958067,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.2,
|
|
"completions/max_terminated_length": 308.2,
|
|
"completions/mean_length": 211.4125,
|
|
"completions/mean_terminated_length": 211.4125,
|
|
"completions/min_length": 135.8,
|
|
"completions/min_terminated_length": 135.8,
|
|
"entropy": 1.0608094729483128,
|
|
"epoch": 0.1581318378229291,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1368068903684616,
|
|
"kl": 0.03478015947039239,
|
|
"learning_rate": 4.999902825999588e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 7481155.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.2917597532272339,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.2917597651481628,
|
|
"sampling/importance_sampling_ratio/max": 1.3059454321861268,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001837849617004,
|
|
"sampling/importance_sampling_ratio/min": 0.7194568514823914,
|
|
"sampling/sampling_logp_difference/max": 0.4368997931480408,
|
|
"sampling/sampling_logp_difference/mean": 0.02122602015733719,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.9,
|
|
"completions/max_terminated_length": 293.9,
|
|
"completions/mean_length": 206.5875,
|
|
"completions/mean_terminated_length": 206.5875,
|
|
"completions/min_length": 131.1,
|
|
"completions/min_terminated_length": 131.1,
|
|
"entropy": 1.0386747267097234,
|
|
"epoch": 0.15905120897306244,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.215572357177734,
|
|
"kl": 0.04280963434721343,
|
|
"learning_rate": 4.9998792034971645e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 7524506.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.3547274947166443,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.3547275006771088,
|
|
"sampling/importance_sampling_ratio/max": 1.281489098072052,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999694466590882,
|
|
"sampling/importance_sampling_ratio/min": 0.7720149755477905,
|
|
"sampling/sampling_logp_difference/max": 0.2822927713394165,
|
|
"sampling/sampling_logp_difference/mean": 0.021051739901304246,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.6,
|
|
"completions/max_terminated_length": 307.6,
|
|
"completions/mean_length": 212.0625,
|
|
"completions/mean_terminated_length": 212.0625,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"entropy": 1.1329169027507304,
|
|
"epoch": 0.15997058012319573,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.036554336547852,
|
|
"kl": 0.033280878106597814,
|
|
"learning_rate": 4.999853013418409e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 7571895.0,
|
|
"reward": -0.3375,
|
|
"reward_std": 0.47455789148807526,
|
|
"rewards/reward_fn/mean": -0.3375,
|
|
"rewards/reward_fn/std": 0.4745579093694687,
|
|
"sampling/importance_sampling_ratio/max": 1.2642733454704285,
|
|
"sampling/importance_sampling_ratio/mean": 1.000291782617569,
|
|
"sampling/importance_sampling_ratio/min": 0.7963159680366516,
|
|
"sampling/sampling_logp_difference/max": 0.24890742301940919,
|
|
"sampling/sampling_logp_difference/mean": 0.02223680019378662,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.1,
|
|
"completions/max_terminated_length": 268.1,
|
|
"completions/mean_length": 187.6125,
|
|
"completions/mean_terminated_length": 187.6125,
|
|
"completions/min_length": 128.6,
|
|
"completions/min_terminated_length": 128.6,
|
|
"entropy": 1.0342995330691338,
|
|
"epoch": 0.16088995127332903,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.907735824584961,
|
|
"kl": 0.03509269545902498,
|
|
"learning_rate": 4.99982425579022e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 7612040.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.4307587772607803,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.4307588070631027,
|
|
"sampling/importance_sampling_ratio/max": 1.2999369382858277,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000859439373015,
|
|
"sampling/importance_sampling_ratio/min": 0.7732944309711456,
|
|
"sampling/sampling_logp_difference/max": 0.28751913607120516,
|
|
"sampling/sampling_logp_difference/mean": 0.021382975578308105,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.1,
|
|
"completions/max_terminated_length": 291.1,
|
|
"completions/mean_length": 210.825,
|
|
"completions/mean_terminated_length": 210.825,
|
|
"completions/min_length": 149.1,
|
|
"completions/min_terminated_length": 149.1,
|
|
"entropy": 1.1162607789039611,
|
|
"epoch": 0.16180932242346235,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.0578179359436035,
|
|
"kl": 0.026344594033434986,
|
|
"learning_rate": 4.999792930642135e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 7654626.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.4763862133026123,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.47638622522354124,
|
|
"sampling/importance_sampling_ratio/max": 1.286192572116852,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006832718849181,
|
|
"sampling/importance_sampling_ratio/min": 0.7646889507770538,
|
|
"sampling/sampling_logp_difference/max": 0.3069689214229584,
|
|
"sampling/sampling_logp_difference/mean": 0.02153454050421715,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 348.0,
|
|
"completions/max_terminated_length": 323.8,
|
|
"completions/mean_length": 227.3375,
|
|
"completions/mean_terminated_length": 223.60535736083983,
|
|
"completions/min_length": 145.3,
|
|
"completions/min_terminated_length": 145.3,
|
|
"entropy": 1.1258797220885755,
|
|
"epoch": 0.16272869357359565,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.659244537353516,
|
|
"kl": 0.021348016592673956,
|
|
"learning_rate": 4.999759038006328e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 7701701.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.41403928101062776,
|
|
"rewards/reward_fn/mean": 0.15625,
|
|
"rewards/reward_fn/std": 0.4140392869710922,
|
|
"sampling/importance_sampling_ratio/max": 1.3567353725433349,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997352540493012,
|
|
"sampling/importance_sampling_ratio/min": 0.7304510235786438,
|
|
"sampling/sampling_logp_difference/max": 0.3939854145050049,
|
|
"sampling/sampling_logp_difference/mean": 0.021448280289769174,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 252.4,
|
|
"completions/max_terminated_length": 252.4,
|
|
"completions/mean_length": 187.225,
|
|
"completions/mean_terminated_length": 187.225,
|
|
"completions/min_length": 129.2,
|
|
"completions/min_terminated_length": 129.2,
|
|
"entropy": 1.1231333188712598,
|
|
"epoch": 0.16364806472372898,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.789471626281738,
|
|
"kl": 0.015499754762277006,
|
|
"learning_rate": 4.9997225779176075e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 7744951.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.36909410953521726,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.36909412145614623,
|
|
"sampling/importance_sampling_ratio/max": 1.2488541722297668,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003271102905273,
|
|
"sampling/importance_sampling_ratio/min": 0.6873427987098694,
|
|
"sampling/sampling_logp_difference/max": 0.45946853756904604,
|
|
"sampling/sampling_logp_difference/mean": 0.0220858970656991,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 253.1,
|
|
"completions/max_terminated_length": 253.1,
|
|
"completions/mean_length": 187.0375,
|
|
"completions/mean_terminated_length": 187.0375,
|
|
"completions/min_length": 125.2,
|
|
"completions/min_terminated_length": 125.2,
|
|
"entropy": 1.0134456224739552,
|
|
"epoch": 0.16456743587386227,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.742940425872803,
|
|
"kl": 0.020215615181950853,
|
|
"learning_rate": 4.999683550413424e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 7783418.0,
|
|
"reward": 0.29375,
|
|
"reward_std": 0.6668038606643677,
|
|
"rewards/reward_fn/mean": 0.29375,
|
|
"rewards/reward_fn/std": 0.6668038606643677,
|
|
"sampling/importance_sampling_ratio/max": 1.299957811832428,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998450696468353,
|
|
"sampling/importance_sampling_ratio/min": 0.7644662320613861,
|
|
"sampling/sampling_logp_difference/max": 0.3087465226650238,
|
|
"sampling/sampling_logp_difference/mean": 0.02149232681840658,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 298.3,
|
|
"completions/max_terminated_length": 296.8,
|
|
"completions/mean_length": 203.375,
|
|
"completions/mean_terminated_length": 200.50357360839843,
|
|
"completions/min_length": 117.8,
|
|
"completions/min_terminated_length": 117.8,
|
|
"entropy": 1.0240340236574412,
|
|
"epoch": 0.1654868070239956,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.15658947825431824,
|
|
"kl": 0.01568760203081183,
|
|
"learning_rate": 4.999641955533858e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 7829024.0,
|
|
"reward": -0.26875,
|
|
"reward_std": 0.20908173173666,
|
|
"rewards/reward_fn/mean": -0.26875,
|
|
"rewards/reward_fn/std": 0.2090817376971245,
|
|
"sampling/importance_sampling_ratio/max": 1.3214632987976074,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997395277023315,
|
|
"sampling/importance_sampling_ratio/min": 0.7777888774871826,
|
|
"sampling/sampling_logp_difference/max": 0.3096400260925293,
|
|
"sampling/sampling_logp_difference/mean": 0.020993327908217906,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.1654868070239956,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0008726003490401396,
|
|
"eval_completions/max_length": 268.956369982548,
|
|
"eval_completions/max_terminated_length": 267.6928446771379,
|
|
"eval_completions/mean_length": 189.07089877835952,
|
|
"eval_completions/mean_terminated_length": 188.78922550008352,
|
|
"eval_completions/min_length": 121.76788830715532,
|
|
"eval_completions/min_terminated_length": 121.76788830715532,
|
|
"eval_entropy": 1.0206009637414888,
|
|
"eval_frac_reward_zero_std": 0.33507853403141363,
|
|
"eval_kl": 0.016771220259147374,
|
|
"eval_loss": 0.0008007879951037467,
|
|
"eval_num_tokens": 7829024.0,
|
|
"eval_reward": -0.033595113438045374,
|
|
"eval_reward_std": 0.4418367014557487,
|
|
"eval_rewards/reward_fn/mean": -0.033595113438045374,
|
|
"eval_rewards/reward_fn/std": 0.441836712690132,
|
|
"eval_runtime": 1591.0901,
|
|
"eval_samples_per_second": 0.36,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2806214154494786,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000317281869486,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7608642992912995,
|
|
"eval_sampling/sampling_logp_difference/max": 0.30833629657876843,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021213743801555395,
|
|
"eval_steps_per_second": 0.045,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 318.3,
|
|
"completions/max_terminated_length": 318.3,
|
|
"completions/mean_length": 220.1625,
|
|
"completions/mean_terminated_length": 220.1625,
|
|
"completions/min_length": 143.9,
|
|
"completions/min_terminated_length": 143.9,
|
|
"entropy": 1.1191354081034661,
|
|
"epoch": 0.1664061781741289,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.302118301391602,
|
|
"kl": 0.019197515666019172,
|
|
"learning_rate": 4.999597793321635e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 7875549.0,
|
|
"reward": -0.35,
|
|
"reward_std": 0.5776334583759308,
|
|
"rewards/reward_fn/mean": -0.35,
|
|
"rewards/reward_fn/std": 0.5776334643363953,
|
|
"sampling/importance_sampling_ratio/max": 1.2857922315597534,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001758277416228,
|
|
"sampling/importance_sampling_ratio/min": 0.7315779685974121,
|
|
"sampling/sampling_logp_difference/max": 0.3537005364894867,
|
|
"sampling/sampling_logp_difference/mean": 0.02154194638133049,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.4,
|
|
"completions/max_terminated_length": 276.4,
|
|
"completions/mean_length": 202.7875,
|
|
"completions/mean_terminated_length": 202.7875,
|
|
"completions/min_length": 141.4,
|
|
"completions/min_terminated_length": 141.4,
|
|
"entropy": 0.9648481421172619,
|
|
"epoch": 0.16732554932426222,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.462383270263672,
|
|
"kl": 0.020946410379838198,
|
|
"learning_rate": 4.9995510638221086e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 7920572.0,
|
|
"reward": -0.475,
|
|
"reward_std": 0.5935844153165817,
|
|
"rewards/reward_fn/mean": -0.475,
|
|
"rewards/reward_fn/std": 0.5935844212770462,
|
|
"sampling/importance_sampling_ratio/max": 1.3535225510597229,
|
|
"sampling/importance_sampling_ratio/mean": 1.000580257177353,
|
|
"sampling/importance_sampling_ratio/min": 0.7677641868591308,
|
|
"sampling/sampling_logp_difference/max": 0.34211663603782655,
|
|
"sampling/sampling_logp_difference/mean": 0.021085841208696367,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.3,
|
|
"completions/max_terminated_length": 302.3,
|
|
"completions/mean_length": 212.3,
|
|
"completions/mean_terminated_length": 212.3,
|
|
"completions/min_length": 132.4,
|
|
"completions/min_terminated_length": 132.4,
|
|
"entropy": 1.0180262207984925,
|
|
"epoch": 0.16824492047439552,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.7959442138671875,
|
|
"kl": 0.030683148233219982,
|
|
"learning_rate": 4.999501767083278e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 7965572.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.5805714815855026,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.5805714935064316,
|
|
"sampling/importance_sampling_ratio/max": 1.3044957280158997,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997461616992951,
|
|
"sampling/importance_sampling_ratio/min": 0.7525037109851838,
|
|
"sampling/sampling_logp_difference/max": 0.3274673104286194,
|
|
"sampling/sampling_logp_difference/mean": 0.02111899834126234,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 300.6,
|
|
"completions/max_terminated_length": 279.2,
|
|
"completions/mean_length": 224.1125,
|
|
"completions/mean_terminated_length": 220.81964416503905,
|
|
"completions/min_length": 154.9,
|
|
"completions/min_terminated_length": 154.9,
|
|
"entropy": 1.122595849633217,
|
|
"epoch": 0.1691642916245288,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.7362141609191895,
|
|
"kl": 0.02047282544663176,
|
|
"learning_rate": 4.999449903155772e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 8009557.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.7461822956800461,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.7461823433637619,
|
|
"sampling/importance_sampling_ratio/max": 1.3128098130226136,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002018332481384,
|
|
"sampling/importance_sampling_ratio/min": 0.7588152348995209,
|
|
"sampling/sampling_logp_difference/max": 0.30289416909217837,
|
|
"sampling/sampling_logp_difference/mean": 0.022252458706498147,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.3,
|
|
"completions/max_terminated_length": 315.3,
|
|
"completions/mean_length": 219.6125,
|
|
"completions/mean_terminated_length": 219.6125,
|
|
"completions/min_length": 143.4,
|
|
"completions/min_terminated_length": 143.4,
|
|
"entropy": 1.1157530531287194,
|
|
"epoch": 0.17008366277466214,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11803943663835526,
|
|
"kl": 0.02386214191792533,
|
|
"learning_rate": 4.999395472092861e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 8053110.0,
|
|
"reward": -0.55,
|
|
"reward_std": 0.41486670076847076,
|
|
"rewards/reward_fn/mean": -0.55,
|
|
"rewards/reward_fn/std": 0.4148667067289352,
|
|
"sampling/importance_sampling_ratio/max": 1.3096130967140198,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998457193374634,
|
|
"sampling/importance_sampling_ratio/min": 0.7483814895153046,
|
|
"sampling/sampling_logp_difference/max": 0.3229205310344696,
|
|
"sampling/sampling_logp_difference/mean": 0.02229196336120367,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.3,
|
|
"completions/max_terminated_length": 295.3,
|
|
"completions/mean_length": 211.675,
|
|
"completions/mean_terminated_length": 211.675,
|
|
"completions/min_length": 145.4,
|
|
"completions/min_terminated_length": 145.4,
|
|
"entropy": 0.9884705692529678,
|
|
"epoch": 0.17100303392479543,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.273815155029297,
|
|
"kl": 0.029997169808484613,
|
|
"learning_rate": 4.999338473950448e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 8092468.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.4584876537322998,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.45848766565322874,
|
|
"sampling/importance_sampling_ratio/max": 1.2812983393669128,
|
|
"sampling/importance_sampling_ratio/mean": 1.000443696975708,
|
|
"sampling/importance_sampling_ratio/min": 0.7582194089889527,
|
|
"sampling/sampling_logp_difference/max": 0.302225387096405,
|
|
"sampling/sampling_logp_difference/mean": 0.02075035348534584,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 360.2,
|
|
"completions/max_terminated_length": 357.7,
|
|
"completions/mean_length": 240.2875,
|
|
"completions/mean_terminated_length": 237.825,
|
|
"completions/min_length": 138.9,
|
|
"completions/min_terminated_length": 138.9,
|
|
"entropy": 1.1315513290464878,
|
|
"epoch": 0.17192240507492876,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.1036810874938965,
|
|
"kl": 0.014520496840123086,
|
|
"learning_rate": 4.9992789087870764e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 8137827.0,
|
|
"reward": 0.14375,
|
|
"reward_std": 0.5457315862178802,
|
|
"rewards/reward_fn/mean": 0.14375,
|
|
"rewards/reward_fn/std": 0.5457316160202026,
|
|
"sampling/importance_sampling_ratio/max": 1.2495078206062318,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996249377727509,
|
|
"sampling/importance_sampling_ratio/min": 0.7946069240570068,
|
|
"sampling/sampling_logp_difference/max": 0.24551107883453369,
|
|
"sampling/sampling_logp_difference/mean": 0.021386440843343735,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.2,
|
|
"completions/max_terminated_length": 269.2,
|
|
"completions/mean_length": 196.55,
|
|
"completions/mean_terminated_length": 196.55,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 1.038776909559965,
|
|
"epoch": 0.17284177622506205,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1355523020029068,
|
|
"kl": 0.018370905792107806,
|
|
"learning_rate": 4.999216776663924e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 8175687.0,
|
|
"reward": 0.525,
|
|
"reward_std": 0.39513265490531924,
|
|
"rewards/reward_fn/mean": 0.525,
|
|
"rewards/reward_fn/std": 0.3951326608657837,
|
|
"sampling/importance_sampling_ratio/max": 1.3396273612976075,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000875353813172,
|
|
"sampling/importance_sampling_ratio/min": 0.7652070045471191,
|
|
"sampling/sampling_logp_difference/max": 0.3233711004257202,
|
|
"sampling/sampling_logp_difference/mean": 0.021475713327527048,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 320.5,
|
|
"completions/max_terminated_length": 320.5,
|
|
"completions/mean_length": 224.575,
|
|
"completions/mean_terminated_length": 224.575,
|
|
"completions/min_length": 142.8,
|
|
"completions/min_terminated_length": 142.8,
|
|
"entropy": 1.1039319053292274,
|
|
"epoch": 0.17376114737519538,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.80249547958374,
|
|
"kl": 0.02133035861188546,
|
|
"learning_rate": 4.999152077644805e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 8220229.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.7276197880506515,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.7276198178529739,
|
|
"sampling/importance_sampling_ratio/max": 1.297184681892395,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996058583259583,
|
|
"sampling/importance_sampling_ratio/min": 0.7719779908657074,
|
|
"sampling/sampling_logp_difference/max": 0.28920464515686034,
|
|
"sampling/sampling_logp_difference/mean": 0.020804373547434807,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.2,
|
|
"completions/max_terminated_length": 273.2,
|
|
"completions/mean_length": 182.775,
|
|
"completions/mean_terminated_length": 182.775,
|
|
"completions/min_length": 119.5,
|
|
"completions/min_terminated_length": 119.5,
|
|
"entropy": 1.1487253315746784,
|
|
"epoch": 0.17468051852532868,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.879741191864014,
|
|
"kl": 0.015241934260120615,
|
|
"learning_rate": 4.9990848117961705e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 8260819.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.40172620117664337,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.40172623097896576,
|
|
"sampling/importance_sampling_ratio/max": 1.2542536377906799,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002043008804322,
|
|
"sampling/importance_sampling_ratio/min": 0.7620341658592225,
|
|
"sampling/sampling_logp_difference/max": 0.2907947599887848,
|
|
"sampling/sampling_logp_difference/mean": 0.022572824731469154,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 329.5,
|
|
"completions/max_terminated_length": 329.5,
|
|
"completions/mean_length": 209.6625,
|
|
"completions/mean_terminated_length": 209.6625,
|
|
"completions/min_length": 100.4,
|
|
"completions/min_terminated_length": 100.4,
|
|
"entropy": 1.107984083145857,
|
|
"epoch": 0.17559988967546197,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.747552871704102,
|
|
"kl": 0.017384014307754114,
|
|
"learning_rate": 4.999014979187108e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 8302392.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.5283277183771133,
|
|
"rewards/reward_fn/mean": 0.4375,
|
|
"rewards/reward_fn/std": 0.5283277422189713,
|
|
"sampling/importance_sampling_ratio/max": 1.2413341641426086,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996440470218658,
|
|
"sampling/importance_sampling_ratio/min": 0.7668880820274353,
|
|
"sampling/sampling_logp_difference/max": 0.276617693901062,
|
|
"sampling/sampling_logp_difference/mean": 0.021286442503333093,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.1,
|
|
"completions/max_terminated_length": 308.1,
|
|
"completions/mean_length": 209.2,
|
|
"completions/mean_terminated_length": 209.2,
|
|
"completions/min_length": 137.3,
|
|
"completions/min_terminated_length": 137.3,
|
|
"entropy": 1.077923546358943,
|
|
"epoch": 0.1765192608255953,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.904787063598633,
|
|
"kl": 0.020174477086402475,
|
|
"learning_rate": 4.9989425798893415e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 8342456.0,
|
|
"reward": -0.5,
|
|
"reward_std": 0.48138763904571535,
|
|
"rewards/reward_fn/mean": -0.5,
|
|
"rewards/reward_fn/std": 0.48138765096664426,
|
|
"sampling/importance_sampling_ratio/max": 1.2854896545410157,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998751997947692,
|
|
"sampling/importance_sampling_ratio/min": 0.7877361714839936,
|
|
"sampling/sampling_logp_difference/max": 0.27977050840854645,
|
|
"sampling/sampling_logp_difference/mean": 0.020843944512307644,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.2,
|
|
"completions/max_terminated_length": 289.2,
|
|
"completions/mean_length": 215.05,
|
|
"completions/mean_terminated_length": 215.05,
|
|
"completions/min_length": 141.6,
|
|
"completions/min_terminated_length": 141.6,
|
|
"entropy": 1.1482290662825108,
|
|
"epoch": 0.1774386319757286,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.440964698791504,
|
|
"kl": 0.017538489843718707,
|
|
"learning_rate": 4.998867613977231e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 8389636.0,
|
|
"reward": -0.00625,
|
|
"reward_std": 0.4986860066652298,
|
|
"rewards/reward_fn/mean": -0.00625,
|
|
"rewards/reward_fn/std": 0.49868602454662325,
|
|
"sampling/importance_sampling_ratio/max": 1.2459492087364197,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001853168010713,
|
|
"sampling/importance_sampling_ratio/min": 0.8000842928886414,
|
|
"sampling/sampling_logp_difference/max": 0.2414076328277588,
|
|
"sampling/sampling_logp_difference/mean": 0.021038925647735594,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 336.4,
|
|
"completions/max_terminated_length": 336.4,
|
|
"completions/mean_length": 233.7,
|
|
"completions/mean_terminated_length": 233.7,
|
|
"completions/min_length": 151.5,
|
|
"completions/min_terminated_length": 151.5,
|
|
"entropy": 1.1576966300606728,
|
|
"epoch": 0.17835800312586192,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.10882382839918137,
|
|
"kl": 0.019720542541472243,
|
|
"learning_rate": 4.998790081527771e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 8433580.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.41069733500480654,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.410697340965271,
|
|
"sampling/importance_sampling_ratio/max": 1.2426087141036988,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995519280433655,
|
|
"sampling/importance_sampling_ratio/min": 0.772708636522293,
|
|
"sampling/sampling_logp_difference/max": 0.26426082849502563,
|
|
"sampling/sampling_logp_difference/mean": 0.021607748977839947,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 322.7,
|
|
"completions/max_terminated_length": 322.7,
|
|
"completions/mean_length": 226.7625,
|
|
"completions/mean_terminated_length": 226.7625,
|
|
"completions/min_length": 143.1,
|
|
"completions/min_terminated_length": 143.1,
|
|
"entropy": 1.116366022080183,
|
|
"epoch": 0.17927737427599522,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.747546195983887,
|
|
"kl": 0.017638581997016445,
|
|
"learning_rate": 4.9987099826205966e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 8479513.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.5626426249742508,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.5626426488161087,
|
|
"sampling/importance_sampling_ratio/max": 1.262166678905487,
|
|
"sampling/importance_sampling_ratio/mean": 1.000122892856598,
|
|
"sampling/importance_sampling_ratio/min": 0.7566198885440827,
|
|
"sampling/sampling_logp_difference/max": 0.2945895254611969,
|
|
"sampling/sampling_logp_difference/mean": 0.02078951820731163,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.5,
|
|
"completions/max_terminated_length": 272.5,
|
|
"completions/mean_length": 194.15,
|
|
"completions/mean_terminated_length": 194.15,
|
|
"completions/min_length": 140.5,
|
|
"completions/min_terminated_length": 140.5,
|
|
"entropy": 1.085212154686451,
|
|
"epoch": 0.18019674542612854,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.095575712621212,
|
|
"kl": 0.03095030563999899,
|
|
"learning_rate": 4.998627317337973e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 8518621.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.3881663382053375,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.3881663382053375,
|
|
"sampling/importance_sampling_ratio/max": 1.3164339065551758,
|
|
"sampling/importance_sampling_ratio/mean": 1.0007083892822266,
|
|
"sampling/importance_sampling_ratio/min": 0.760808676481247,
|
|
"sampling/sampling_logp_difference/max": 0.33363351225852966,
|
|
"sampling/sampling_logp_difference/mean": 0.02128539700061083,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.6,
|
|
"completions/max_terminated_length": 302.6,
|
|
"completions/mean_length": 223.075,
|
|
"completions/mean_terminated_length": 223.075,
|
|
"completions/min_length": 152.3,
|
|
"completions/min_terminated_length": 152.3,
|
|
"entropy": 1.1523349031805992,
|
|
"epoch": 0.18111611657626184,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.747213363647461,
|
|
"kl": 0.02937217736034654,
|
|
"learning_rate": 4.9985420857648044e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 8568403.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.4174750655889511,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.4174750715494156,
|
|
"sampling/importance_sampling_ratio/max": 1.3106306672096253,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993778169155121,
|
|
"sampling/importance_sampling_ratio/min": 0.7692589163780212,
|
|
"sampling/sampling_logp_difference/max": 0.3071877956390381,
|
|
"sampling/sampling_logp_difference/mean": 0.021547301672399044,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.3,
|
|
"completions/max_terminated_length": 293.3,
|
|
"completions/mean_length": 216.2125,
|
|
"completions/mean_terminated_length": 216.2125,
|
|
"completions/min_length": 161.6,
|
|
"completions/min_terminated_length": 161.6,
|
|
"entropy": 1.0855547845363618,
|
|
"epoch": 0.18203548772639513,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.09698199480772018,
|
|
"kl": 0.022887393622659145,
|
|
"learning_rate": 4.998454287988632e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 8610244.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.5482046812772751,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.5482046931982041,
|
|
"sampling/importance_sampling_ratio/max": 1.2999603390693664,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996606171131134,
|
|
"sampling/importance_sampling_ratio/min": 0.7765462398529053,
|
|
"sampling/sampling_logp_difference/max": 0.30179101824760435,
|
|
"sampling/sampling_logp_difference/mean": 0.020937790907919408,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.1,
|
|
"completions/max_terminated_length": 289.1,
|
|
"completions/mean_length": 209.0375,
|
|
"completions/mean_terminated_length": 209.0375,
|
|
"completions/min_length": 152.1,
|
|
"completions/min_terminated_length": 152.1,
|
|
"entropy": 1.029877895116806,
|
|
"epoch": 0.18295485887652846,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.12309105694293976,
|
|
"kl": 0.0291645442659501,
|
|
"learning_rate": 4.998363924099629e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 8654591.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.36818075478076934,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.36818076074123385,
|
|
"sampling/importance_sampling_ratio/max": 1.2720123529434204,
|
|
"sampling/importance_sampling_ratio/mean": 0.999754011631012,
|
|
"sampling/importance_sampling_ratio/min": 0.7930947542190552,
|
|
"sampling/sampling_logp_difference/max": 0.25222728550434115,
|
|
"sampling/sampling_logp_difference/mean": 0.020741439983248712,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.2,
|
|
"completions/max_terminated_length": 258.2,
|
|
"completions/mean_length": 195.075,
|
|
"completions/mean_terminated_length": 195.075,
|
|
"completions/min_length": 145.4,
|
|
"completions/min_terminated_length": 145.4,
|
|
"entropy": 0.9862498935312033,
|
|
"epoch": 0.18387423002666176,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.373353004455566,
|
|
"kl": 0.03954175267135725,
|
|
"learning_rate": 4.998270994190609e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 8697589.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.4119541674852371,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.4119541734457016,
|
|
"sampling/importance_sampling_ratio/max": 1.2410858035087586,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996942818164826,
|
|
"sampling/importance_sampling_ratio/min": 0.7882209658622742,
|
|
"sampling/sampling_logp_difference/max": 0.2585688352584839,
|
|
"sampling/sampling_logp_difference/mean": 0.020708341896533967,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.18387423002666176,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002181500872600349,
|
|
"eval_completions/max_length": 294.13961605584643,
|
|
"eval_completions/max_terminated_length": 292.0069808027923,
|
|
"eval_completions/mean_length": 209.1326352530541,
|
|
"eval_completions/mean_terminated_length": 208.5953214156066,
|
|
"eval_completions/min_length": 141.1064572425829,
|
|
"eval_completions/min_terminated_length": 141.1064572425829,
|
|
"eval_entropy": 1.0833796258060095,
|
|
"eval_frac_reward_zero_std": 0.23734729493891799,
|
|
"eval_kl": 0.037226662446092976,
|
|
"eval_loss": 0.0019072559662163258,
|
|
"eval_num_tokens": 8697589.0,
|
|
"eval_reward": 0.00981675392670157,
|
|
"eval_reward_std": 0.4129836211861948,
|
|
"eval_rewards/reward_fn/mean": 0.00981675392670157,
|
|
"eval_rewards/reward_fn/std": 0.41298363491710777,
|
|
"eval_runtime": 1720.6011,
|
|
"eval_samples_per_second": 0.333,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2999582694255036,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999931461940053,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7586964579724933,
|
|
"eval_sampling/sampling_logp_difference/max": 0.32477144077810316,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021163378978703972,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 265.1,
|
|
"completions/max_terminated_length": 265.1,
|
|
"completions/mean_length": 204.45,
|
|
"completions/mean_terminated_length": 204.45,
|
|
"completions/min_length": 139.5,
|
|
"completions/min_terminated_length": 139.5,
|
|
"entropy": 1.0727780736982822,
|
|
"epoch": 0.18479360117679508,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.539535045623779,
|
|
"kl": 0.05420771196950227,
|
|
"learning_rate": 4.9981754983570165e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 8739433.0,
|
|
"reward": -0.225,
|
|
"reward_std": 0.3548214554786682,
|
|
"rewards/reward_fn/mean": -0.225,
|
|
"rewards/reward_fn/std": 0.3548214614391327,
|
|
"sampling/importance_sampling_ratio/max": 1.3256034851074219,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996547698974609,
|
|
"sampling/importance_sampling_ratio/min": 0.7573504507541656,
|
|
"sampling/sampling_logp_difference/max": 0.3263516664505005,
|
|
"sampling/sampling_logp_difference/mean": 0.021330700255930424,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.8,
|
|
"completions/max_terminated_length": 289.8,
|
|
"completions/mean_length": 208.7875,
|
|
"completions/mean_terminated_length": 208.7875,
|
|
"completions/min_length": 132.2,
|
|
"completions/min_terminated_length": 132.2,
|
|
"entropy": 1.1074910007417202,
|
|
"epoch": 0.18571297232692838,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1339806318283081,
|
|
"kl": 0.04188211368164048,
|
|
"learning_rate": 4.998077436696934e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 8783264.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.3818357288837433,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.38183574080467225,
|
|
"sampling/importance_sampling_ratio/max": 1.2332355976104736,
|
|
"sampling/importance_sampling_ratio/mean": 1.000568288564682,
|
|
"sampling/importance_sampling_ratio/min": 0.7717979192733765,
|
|
"sampling/sampling_logp_difference/max": 0.2694129407405853,
|
|
"sampling/sampling_logp_difference/mean": 0.02108699418604374,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.4,
|
|
"completions/max_terminated_length": 276.4,
|
|
"completions/mean_length": 205.95,
|
|
"completions/mean_terminated_length": 205.95,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 1.0832145676016807,
|
|
"epoch": 0.1866323434770617,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.657064437866211,
|
|
"kl": 0.054080014571081844,
|
|
"learning_rate": 4.997976809311078e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 8824580.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.2328435957431793,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.2328436017036438,
|
|
"sampling/importance_sampling_ratio/max": 1.3020219087600708,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002191722393037,
|
|
"sampling/importance_sampling_ratio/min": 0.7339301750063896,
|
|
"sampling/sampling_logp_difference/max": 0.3908567070960999,
|
|
"sampling/sampling_logp_difference/mean": 0.02179529834538698,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.6,
|
|
"completions/max_terminated_length": 269.6,
|
|
"completions/mean_length": 186.1,
|
|
"completions/mean_terminated_length": 186.1,
|
|
"completions/min_length": 115.9,
|
|
"completions/min_terminated_length": 115.9,
|
|
"entropy": 1.0329186499118805,
|
|
"epoch": 0.187551714627195,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.15627823770046234,
|
|
"kl": 0.03850859795929864,
|
|
"learning_rate": 4.997873616302802e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 8866460.0,
|
|
"reward": 0.25625,
|
|
"reward_std": 0.3145385950803757,
|
|
"rewards/reward_fn/mean": 0.25625,
|
|
"rewards/reward_fn/std": 0.31453860104084014,
|
|
"sampling/importance_sampling_ratio/max": 1.2636144042015076,
|
|
"sampling/importance_sampling_ratio/mean": 1.000742393732071,
|
|
"sampling/importance_sampling_ratio/min": 0.7930640876293182,
|
|
"sampling/sampling_logp_difference/max": 0.2515856772661209,
|
|
"sampling/sampling_logp_difference/mean": 0.02052618097513914,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.0,
|
|
"completions/max_terminated_length": 311.0,
|
|
"completions/mean_length": 220.4,
|
|
"completions/mean_terminated_length": 220.4,
|
|
"completions/min_length": 133.7,
|
|
"completions/min_terminated_length": 133.7,
|
|
"entropy": 1.055208795145154,
|
|
"epoch": 0.1884710857773283,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.259217739105225,
|
|
"kl": 0.03475877782329917,
|
|
"learning_rate": 4.997767857778093e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 8912868.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.3383516758680344,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.3383516877889633,
|
|
"sampling/importance_sampling_ratio/max": 1.283850336074829,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002785205841065,
|
|
"sampling/importance_sampling_ratio/min": 0.7336234390735626,
|
|
"sampling/sampling_logp_difference/max": 0.33705209493637084,
|
|
"sampling/sampling_logp_difference/mean": 0.021786217391490937,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.3,
|
|
"completions/max_terminated_length": 306.3,
|
|
"completions/mean_length": 225.3875,
|
|
"completions/mean_terminated_length": 225.3875,
|
|
"completions/min_length": 157.2,
|
|
"completions/min_terminated_length": 157.2,
|
|
"entropy": 1.0984139777719975,
|
|
"epoch": 0.18939045692746162,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13372617959976196,
|
|
"kl": 0.027556859265314416,
|
|
"learning_rate": 4.997659533845574e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 8959131.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.2394672751426697,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.23946728110313414,
|
|
"sampling/importance_sampling_ratio/max": 1.242374837398529,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999833285808564,
|
|
"sampling/importance_sampling_ratio/min": 0.7497252464294434,
|
|
"sampling/sampling_logp_difference/max": 0.29617769122123716,
|
|
"sampling/sampling_logp_difference/mean": 0.021633453853428363,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.6,
|
|
"completions/max_terminated_length": 307.6,
|
|
"completions/mean_length": 222.7125,
|
|
"completions/mean_terminated_length": 222.7125,
|
|
"completions/min_length": 154.1,
|
|
"completions/min_terminated_length": 154.1,
|
|
"entropy": 1.066904890537262,
|
|
"epoch": 0.19030982807759492,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.10214605182409286,
|
|
"kl": 0.04522109412937425,
|
|
"learning_rate": 4.997548644616501e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 9002820.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.3167852282524109,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.3167852461338043,
|
|
"sampling/importance_sampling_ratio/max": 1.280836009979248,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996492743492127,
|
|
"sampling/importance_sampling_ratio/min": 0.7761459469795227,
|
|
"sampling/sampling_logp_difference/max": 0.2953964829444885,
|
|
"sampling/sampling_logp_difference/mean": 0.021134966053068638,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 332.4,
|
|
"completions/max_terminated_length": 332.4,
|
|
"completions/mean_length": 245.9125,
|
|
"completions/mean_terminated_length": 245.9125,
|
|
"completions/min_length": 171.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"entropy": 1.1535887710750103,
|
|
"epoch": 0.19122919922772824,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.10864507406949997,
|
|
"kl": 0.04408502578153275,
|
|
"learning_rate": 4.997435190204767e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 9050621.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.2856272906064987,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.2856272965669632,
|
|
"sampling/importance_sampling_ratio/max": 1.27427978515625,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002832472324372,
|
|
"sampling/importance_sampling_ratio/min": 0.7479039609432221,
|
|
"sampling/sampling_logp_difference/max": 0.32640727460384367,
|
|
"sampling/sampling_logp_difference/mean": 0.021294852904975414,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.2,
|
|
"completions/max_terminated_length": 304.2,
|
|
"completions/mean_length": 216.5625,
|
|
"completions/mean_terminated_length": 216.5625,
|
|
"completions/min_length": 147.4,
|
|
"completions/min_terminated_length": 147.4,
|
|
"entropy": 1.1119799844920635,
|
|
"epoch": 0.19214857037786154,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.983915328979492,
|
|
"kl": 0.045881410402944314,
|
|
"learning_rate": 4.9973191707268986e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 9098066.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.39642935395240786,
|
|
"rewards/reward_fn/mean": 0.2875,
|
|
"rewards/reward_fn/std": 0.3964293718338013,
|
|
"sampling/importance_sampling_ratio/max": 1.2645806550979615,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998108804225921,
|
|
"sampling/importance_sampling_ratio/min": 0.7577556669712067,
|
|
"sampling/sampling_logp_difference/max": 0.30794984102249146,
|
|
"sampling/sampling_logp_difference/mean": 0.02104753293097019,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.5,
|
|
"completions/max_terminated_length": 315.5,
|
|
"completions/mean_length": 234.3125,
|
|
"completions/mean_terminated_length": 234.3125,
|
|
"completions/min_length": 153.5,
|
|
"completions/min_terminated_length": 153.5,
|
|
"entropy": 1.180136325210333,
|
|
"epoch": 0.19306794152799486,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.488556385040283,
|
|
"kl": 0.03945447752485052,
|
|
"learning_rate": 4.9972005863020574e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 9146147.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.47202224731445314,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.47202226519584656,
|
|
"sampling/importance_sampling_ratio/max": 1.380826508998871,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002896964550019,
|
|
"sampling/importance_sampling_ratio/min": 0.792996472120285,
|
|
"sampling/sampling_logp_difference/max": 0.3363222122192383,
|
|
"sampling/sampling_logp_difference/mean": 0.021930642798542975,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.1,
|
|
"completions/max_terminated_length": 298.1,
|
|
"completions/mean_length": 223.3375,
|
|
"completions/mean_terminated_length": 223.3375,
|
|
"completions/min_length": 160.7,
|
|
"completions/min_terminated_length": 160.7,
|
|
"entropy": 1.12664475440979,
|
|
"epoch": 0.19398731267812816,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 3.8552494049072266,
|
|
"kl": 0.028041312610730527,
|
|
"learning_rate": 4.997079437052039e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 9191158.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.3670209854841232,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.3670209974050522,
|
|
"sampling/importance_sampling_ratio/max": 1.3053725361824036,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995544373989105,
|
|
"sampling/importance_sampling_ratio/min": 0.7901838839054107,
|
|
"sampling/sampling_logp_difference/max": 0.2744863986968994,
|
|
"sampling/sampling_logp_difference/mean": 0.021696880273520947,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.1,
|
|
"completions/max_terminated_length": 314.1,
|
|
"completions/mean_length": 242.4625,
|
|
"completions/mean_terminated_length": 242.4625,
|
|
"completions/min_length": 174.8,
|
|
"completions/min_terminated_length": 174.8,
|
|
"entropy": 1.1719475172460079,
|
|
"epoch": 0.19490668382826146,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.561178207397461,
|
|
"kl": 0.039780617982614785,
|
|
"learning_rate": 4.996955723101273e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 9239419.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.4653678238391876,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.46536784172058104,
|
|
"sampling/importance_sampling_ratio/max": 1.3244190573692323,
|
|
"sampling/importance_sampling_ratio/mean": 0.999659663438797,
|
|
"sampling/importance_sampling_ratio/min": 0.7307224631309509,
|
|
"sampling/sampling_logp_difference/max": 0.3514132142066956,
|
|
"sampling/sampling_logp_difference/mean": 0.0220487168058753,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 321.7,
|
|
"completions/max_terminated_length": 321.7,
|
|
"completions/mean_length": 241.0625,
|
|
"completions/mean_terminated_length": 241.0625,
|
|
"completions/min_length": 161.0,
|
|
"completions/min_terminated_length": 161.0,
|
|
"entropy": 1.1713617824018001,
|
|
"epoch": 0.19582605497839478,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.935511589050293,
|
|
"kl": 0.035165926109766585,
|
|
"learning_rate": 4.996829444576825e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 9288672.0,
|
|
"reward": -0.2375,
|
|
"reward_std": 0.34120489954948424,
|
|
"rewards/reward_fn/mean": -0.2375,
|
|
"rewards/reward_fn/std": 0.34120491743087766,
|
|
"sampling/importance_sampling_ratio/max": 1.2524248003959655,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999395251274109,
|
|
"sampling/importance_sampling_ratio/min": 0.7554161071777343,
|
|
"sampling/sampling_logp_difference/max": 0.312109237909317,
|
|
"sampling/sampling_logp_difference/mean": 0.020913349092006685,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 323.8,
|
|
"completions/max_terminated_length": 323.8,
|
|
"completions/mean_length": 243.2,
|
|
"completions/mean_terminated_length": 243.2,
|
|
"completions/min_length": 158.3,
|
|
"completions/min_terminated_length": 158.3,
|
|
"entropy": 1.1296326026320458,
|
|
"epoch": 0.19674542612852808,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.870914936065674,
|
|
"kl": 0.04121863882755861,
|
|
"learning_rate": 4.996700601608391e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 9335976.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.3774735927581787,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.37747361660003664,
|
|
"sampling/importance_sampling_ratio/max": 1.2856929779052735,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002261340618133,
|
|
"sampling/importance_sampling_ratio/min": 0.7678008794784545,
|
|
"sampling/sampling_logp_difference/max": 0.2914842367172241,
|
|
"sampling/sampling_logp_difference/mean": 0.02136395815759897,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.6,
|
|
"completions/max_terminated_length": 304.6,
|
|
"completions/mean_length": 209.0625,
|
|
"completions/mean_terminated_length": 209.0625,
|
|
"completions/min_length": 130.5,
|
|
"completions/min_terminated_length": 130.5,
|
|
"entropy": 1.2393813714385034,
|
|
"epoch": 0.1976647972786614,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.041493892669678,
|
|
"kl": 0.037880709511227906,
|
|
"learning_rate": 4.9965691943283045e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 9378029.0,
|
|
"reward": -0.20625,
|
|
"reward_std": 0.29540800750255586,
|
|
"rewards/reward_fn/mean": -0.20625,
|
|
"rewards/reward_fn/std": 0.29540803134441374,
|
|
"sampling/importance_sampling_ratio/max": 1.2303287863731385,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006081283092498,
|
|
"sampling/importance_sampling_ratio/min": 0.7902114391326904,
|
|
"sampling/sampling_logp_difference/max": 0.24944406747817993,
|
|
"sampling/sampling_logp_difference/mean": 0.022588260658085346,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.3,
|
|
"completions/max_terminated_length": 294.3,
|
|
"completions/mean_length": 217.4125,
|
|
"completions/mean_terminated_length": 217.4125,
|
|
"completions/min_length": 139.5,
|
|
"completions/min_terminated_length": 139.5,
|
|
"entropy": 1.1623989813029767,
|
|
"epoch": 0.1985841684287947,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.053800582885742,
|
|
"kl": 0.04897128135198727,
|
|
"learning_rate": 4.996435222871531e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 9424854.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.29839127957820893,
|
|
"rewards/reward_fn/mean": 0.3375,
|
|
"rewards/reward_fn/std": 0.29839127957820893,
|
|
"sampling/importance_sampling_ratio/max": 1.2595209956169129,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999686002731323,
|
|
"sampling/importance_sampling_ratio/min": 0.7652659296989441,
|
|
"sampling/sampling_logp_difference/max": 0.29065966606140137,
|
|
"sampling/sampling_logp_difference/mean": 0.021971743553876877,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.1,
|
|
"completions/max_terminated_length": 282.1,
|
|
"completions/mean_length": 196.2375,
|
|
"completions/mean_terminated_length": 196.2375,
|
|
"completions/min_length": 122.0,
|
|
"completions/min_terminated_length": 122.0,
|
|
"entropy": 1.12191246971488,
|
|
"epoch": 0.19950353957892802,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.84433650970459,
|
|
"kl": 0.034589039994170886,
|
|
"learning_rate": 4.996298687375669e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 9464745.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3570065200328827,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.35700652599334715,
|
|
"sampling/importance_sampling_ratio/max": 1.2351934671401978,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003180503845215,
|
|
"sampling/importance_sampling_ratio/min": 0.7983351945877075,
|
|
"sampling/sampling_logp_difference/max": 0.23011383414268494,
|
|
"sampling/sampling_logp_difference/mean": 0.02162060271948576,
|
|
"step": 2170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 244.7,
|
|
"completions/max_terminated_length": 244.7,
|
|
"completions/mean_length": 169.0875,
|
|
"completions/mean_terminated_length": 169.0875,
|
|
"completions/min_length": 114.0,
|
|
"completions/min_terminated_length": 114.0,
|
|
"entropy": 1.080114208161831,
|
|
"epoch": 0.20042291072906132,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12924686074256897,
|
|
"kl": 0.04256691740592942,
|
|
"learning_rate": 4.996159587980953e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 9501200.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.3474793940782547,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.34747940599918364,
|
|
"sampling/importance_sampling_ratio/max": 1.2207862496376038,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995071887969971,
|
|
"sampling/importance_sampling_ratio/min": 0.775526350736618,
|
|
"sampling/sampling_logp_difference/max": 0.2593968629837036,
|
|
"sampling/sampling_logp_difference/mean": 0.02124457973986864,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 340.4,
|
|
"completions/max_terminated_length": 340.4,
|
|
"completions/mean_length": 218.15,
|
|
"completions/mean_terminated_length": 218.15,
|
|
"completions/min_length": 146.1,
|
|
"completions/min_terminated_length": 146.1,
|
|
"entropy": 1.096110636740923,
|
|
"epoch": 0.20134228187919462,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14402176439762115,
|
|
"kl": 0.03449008559109643,
|
|
"learning_rate": 4.9960179248302485e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 9545660.0,
|
|
"reward": -0.375,
|
|
"reward_std": 0.3575767487287521,
|
|
"rewards/reward_fn/mean": -0.375,
|
|
"rewards/reward_fn/std": 0.3575767546892166,
|
|
"sampling/importance_sampling_ratio/max": 1.2408198356628417,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994823038578033,
|
|
"sampling/importance_sampling_ratio/min": 0.7583486199378967,
|
|
"sampling/sampling_logp_difference/max": 0.2966102182865143,
|
|
"sampling/sampling_logp_difference/mean": 0.020903543941676615,
|
|
"step": 2190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.1,
|
|
"completions/max_terminated_length": 280.1,
|
|
"completions/mean_length": 187.3125,
|
|
"completions/mean_terminated_length": 187.3125,
|
|
"completions/min_length": 114.7,
|
|
"completions/min_terminated_length": 114.7,
|
|
"entropy": 0.9106440592557192,
|
|
"epoch": 0.20226165302932794,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.621983051300049,
|
|
"kl": 0.04620505248894915,
|
|
"learning_rate": 4.995873698069054e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 9584253.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.34637176990509033,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.3463717758655548,
|
|
"sampling/importance_sampling_ratio/max": 1.3062060832977296,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998080670833588,
|
|
"sampling/importance_sampling_ratio/min": 0.766309243440628,
|
|
"sampling/sampling_logp_difference/max": 0.30480992794036865,
|
|
"sampling/sampling_logp_difference/mean": 0.01946581043303013,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 0.20226165302932794,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0004363001745200698,
|
|
"eval_completions/max_length": 270.21465968586386,
|
|
"eval_completions/max_terminated_length": 269.76439790575915,
|
|
"eval_completions/mean_length": 186.1173647469459,
|
|
"eval_completions/mean_terminated_length": 185.99071306957623,
|
|
"eval_completions/min_length": 116.75043630017451,
|
|
"eval_completions/min_terminated_length": 116.75043630017451,
|
|
"eval_entropy": 1.0583471444160735,
|
|
"eval_frac_reward_zero_std": 0.27225130890052357,
|
|
"eval_kl": 0.03981750488547806,
|
|
"eval_loss": 0.0020967598538845778,
|
|
"eval_num_tokens": 9584253.0,
|
|
"eval_reward": 0.029668411867364748,
|
|
"eval_reward_std": 0.39895731117088756,
|
|
"eval_rewards/reward_fn/mean": 0.029668411867364748,
|
|
"eval_rewards/reward_fn/std": 0.3989573223012488,
|
|
"eval_runtime": 1597.1237,
|
|
"eval_samples_per_second": 0.359,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.275554472863362,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999324976877808,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7752521586771827,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2901335429874806,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020724559201164915,
|
|
"eval_steps_per_second": 0.045,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.6,
|
|
"completions/max_terminated_length": 299.6,
|
|
"completions/mean_length": 200.3,
|
|
"completions/mean_terminated_length": 200.3,
|
|
"completions/min_length": 135.0,
|
|
"completions/min_terminated_length": 135.0,
|
|
"entropy": 1.0624324716627598,
|
|
"epoch": 0.20318102417946124,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.511288166046143,
|
|
"kl": 0.035200098413042724,
|
|
"learning_rate": 4.995726907845505e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 9626693.0,
|
|
"reward": -0.3125,
|
|
"reward_std": 0.42686644196510315,
|
|
"rewards/reward_fn/mean": -0.3125,
|
|
"rewards/reward_fn/std": 0.4268664479255676,
|
|
"sampling/importance_sampling_ratio/max": 1.2750413417816162,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001168131828309,
|
|
"sampling/importance_sampling_ratio/min": 0.7514230638742447,
|
|
"sampling/sampling_logp_difference/max": 0.31315724849700927,
|
|
"sampling/sampling_logp_difference/mean": 0.021705342829227446,
|
|
"step": 2210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.1,
|
|
"completions/max_terminated_length": 283.1,
|
|
"completions/mean_length": 190.1,
|
|
"completions/mean_terminated_length": 190.1,
|
|
"completions/min_length": 128.4,
|
|
"completions/min_terminated_length": 128.4,
|
|
"entropy": 1.0477397292852402,
|
|
"epoch": 0.20410039532959456,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.10544715076684952,
|
|
"kl": 0.031700442923465744,
|
|
"learning_rate": 4.995577554310363e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 9668269.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.3466205418109894,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.3466205596923828,
|
|
"sampling/importance_sampling_ratio/max": 1.2567079782485961,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003982841968537,
|
|
"sampling/importance_sampling_ratio/min": 0.73931904733181,
|
|
"sampling/sampling_logp_difference/max": 0.3210598468780518,
|
|
"sampling/sampling_logp_difference/mean": 0.020927842892706394,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 278.0,
|
|
"completions/max_terminated_length": 277.3,
|
|
"completions/mean_length": 187.125,
|
|
"completions/mean_terminated_length": 181.6041687011719,
|
|
"completions/min_length": 110.1,
|
|
"completions/min_terminated_length": 110.1,
|
|
"entropy": 0.9990961596369743,
|
|
"epoch": 0.20501976647972786,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.903346538543701,
|
|
"kl": 0.03451996381045319,
|
|
"learning_rate": 4.995425637617027e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 9708199.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.5238449484109878,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.5238449543714523,
|
|
"sampling/importance_sampling_ratio/max": 1.237573754787445,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002052724361419,
|
|
"sampling/importance_sampling_ratio/min": 0.7941884994506836,
|
|
"sampling/sampling_logp_difference/max": 0.247905957698822,
|
|
"sampling/sampling_logp_difference/mean": 0.020801369659602643,
|
|
"step": 2230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.3,
|
|
"completions/max_terminated_length": 276.3,
|
|
"completions/mean_length": 191.15,
|
|
"completions/mean_terminated_length": 191.15,
|
|
"completions/min_length": 124.9,
|
|
"completions/min_terminated_length": 124.9,
|
|
"entropy": 1.10877006649971,
|
|
"epoch": 0.20593913762986119,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.24242615699768066,
|
|
"kl": 0.033766067633405325,
|
|
"learning_rate": 4.995271157921529e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 9748195.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.46596879661083224,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.46596881449222566,
|
|
"sampling/importance_sampling_ratio/max": 1.2661152005195617,
|
|
"sampling/importance_sampling_ratio/mean": 0.999598640203476,
|
|
"sampling/importance_sampling_ratio/min": 0.7782447814941407,
|
|
"sampling/sampling_logp_difference/max": 0.2654001295566559,
|
|
"sampling/sampling_logp_difference/mean": 0.022385939210653304,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 260.2,
|
|
"completions/max_terminated_length": 260.2,
|
|
"completions/mean_length": 186.35,
|
|
"completions/mean_terminated_length": 186.35,
|
|
"completions/min_length": 104.4,
|
|
"completions/min_terminated_length": 104.4,
|
|
"entropy": 1.0151292737573385,
|
|
"epoch": 0.20685850877999448,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.4661545753479,
|
|
"kl": 0.06271298103383742,
|
|
"learning_rate": 4.995114115382532e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 9790023.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.4487059205770493,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.4487059205770493,
|
|
"sampling/importance_sampling_ratio/max": 1.281790828704834,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002649962902068,
|
|
"sampling/importance_sampling_ratio/min": 0.7612417340278625,
|
|
"sampling/sampling_logp_difference/max": 0.2945103049278259,
|
|
"sampling/sampling_logp_difference/mean": 0.020727747585624456,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 212.8,
|
|
"completions/max_terminated_length": 212.8,
|
|
"completions/mean_length": 158.3125,
|
|
"completions/mean_terminated_length": 158.3125,
|
|
"completions/min_length": 104.9,
|
|
"completions/min_terminated_length": 104.9,
|
|
"entropy": 0.9757637903094292,
|
|
"epoch": 0.20777787993012778,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.1468634605407715,
|
|
"kl": 0.03781939910841174,
|
|
"learning_rate": 4.994954510161331e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 9825448.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.36428422927856446,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.36428422927856446,
|
|
"sampling/importance_sampling_ratio/max": 1.2629591584205628,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006554782390595,
|
|
"sampling/importance_sampling_ratio/min": 0.7807676434516907,
|
|
"sampling/sampling_logp_difference/max": 0.26751006841659547,
|
|
"sampling/sampling_logp_difference/mean": 0.021309599094092847,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 253.1,
|
|
"completions/max_terminated_length": 253.1,
|
|
"completions/mean_length": 187.125,
|
|
"completions/mean_terminated_length": 187.125,
|
|
"completions/min_length": 127.8,
|
|
"completions/min_terminated_length": 127.8,
|
|
"entropy": 1.0190150149166584,
|
|
"epoch": 0.2086972510802611,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.6369476318359375,
|
|
"kl": 0.028186429140623658,
|
|
"learning_rate": 4.994792342421852e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 9866826.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.3279952198266983,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.32799523174762724,
|
|
"sampling/importance_sampling_ratio/max": 1.2821413159370423,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998392701148987,
|
|
"sampling/importance_sampling_ratio/min": 0.7528982877731323,
|
|
"sampling/sampling_logp_difference/max": 0.3191342711448669,
|
|
"sampling/sampling_logp_difference/mean": 0.021119408123195172,
|
|
"step": 2270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 321.9,
|
|
"completions/max_terminated_length": 321.9,
|
|
"completions/mean_length": 227.5375,
|
|
"completions/mean_terminated_length": 227.5375,
|
|
"completions/min_length": 142.2,
|
|
"completions/min_terminated_length": 142.2,
|
|
"entropy": 1.0744174726307392,
|
|
"epoch": 0.2096166222303944,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.216409206390381,
|
|
"kl": 0.034819805121514946,
|
|
"learning_rate": 4.994627612330655e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 9916917.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.4910866290330887,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4910866469144821,
|
|
"sampling/importance_sampling_ratio/max": 1.2545270442962646,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997605562210083,
|
|
"sampling/importance_sampling_ratio/min": 0.7645127654075623,
|
|
"sampling/sampling_logp_difference/max": 0.275495845079422,
|
|
"sampling/sampling_logp_difference/mean": 0.020799391716718674,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.3,
|
|
"completions/max_terminated_length": 305.3,
|
|
"completions/mean_length": 216.8875,
|
|
"completions/mean_terminated_length": 216.8875,
|
|
"completions/min_length": 153.7,
|
|
"completions/min_terminated_length": 153.7,
|
|
"entropy": 1.061875542998314,
|
|
"epoch": 0.21053599338052773,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.102997779846191,
|
|
"kl": 0.02847459077311214,
|
|
"learning_rate": 4.994460320056932e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 9961100.0,
|
|
"reward": 0.00625,
|
|
"reward_std": 0.5160910218954087,
|
|
"rewards/reward_fn/mean": 0.00625,
|
|
"rewards/reward_fn/std": 0.5160910278558731,
|
|
"sampling/importance_sampling_ratio/max": 1.2617968320846558,
|
|
"sampling/importance_sampling_ratio/mean": 1.000396978855133,
|
|
"sampling/importance_sampling_ratio/min": 0.7838475465774536,
|
|
"sampling/sampling_logp_difference/max": 0.2614006817340851,
|
|
"sampling/sampling_logp_difference/mean": 0.021053414791822433,
|
|
"step": 2290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.0,
|
|
"completions/max_terminated_length": 313.0,
|
|
"completions/mean_length": 218.1375,
|
|
"completions/mean_terminated_length": 218.1375,
|
|
"completions/min_length": 142.1,
|
|
"completions/min_terminated_length": 142.1,
|
|
"entropy": 1.0973895706236363,
|
|
"epoch": 0.21145536453066102,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.394501686096191,
|
|
"kl": 0.03340508458204568,
|
|
"learning_rate": 4.9942904657725044e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 10004855.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.5268197923898696,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.5268198102712631,
|
|
"sampling/importance_sampling_ratio/max": 1.2829238653182984,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002897024154662,
|
|
"sampling/importance_sampling_ratio/min": 0.7752999007701874,
|
|
"sampling/sampling_logp_difference/max": 0.2683062791824341,
|
|
"sampling/sampling_logp_difference/mean": 0.02157579008489847,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.6,
|
|
"completions/max_terminated_length": 296.6,
|
|
"completions/mean_length": 217.2,
|
|
"completions/mean_terminated_length": 217.2,
|
|
"completions/min_length": 158.1,
|
|
"completions/min_terminated_length": 158.1,
|
|
"entropy": 1.1110787563025952,
|
|
"epoch": 0.21237473568079435,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.3878493309021,
|
|
"kl": 0.03351037814863957,
|
|
"learning_rate": 4.9941180496518265e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 10052295.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.3909654438495636,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.3909654557704926,
|
|
"sampling/importance_sampling_ratio/max": 1.2621555209159852,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002065360546113,
|
|
"sampling/importance_sampling_ratio/min": 0.7889642417430878,
|
|
"sampling/sampling_logp_difference/max": 0.2540652811527252,
|
|
"sampling/sampling_logp_difference/mean": 0.02103464026004076,
|
|
"step": 2310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 343.0,
|
|
"completions/max_terminated_length": 343.0,
|
|
"completions/mean_length": 232.7875,
|
|
"completions/mean_terminated_length": 232.7875,
|
|
"completions/min_length": 151.2,
|
|
"completions/min_terminated_length": 151.2,
|
|
"entropy": 1.1603176042437553,
|
|
"epoch": 0.21329410683092764,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.886568546295166,
|
|
"kl": 0.034893020521849394,
|
|
"learning_rate": 4.993943071871984e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 10098510.0,
|
|
"reward": -0.225,
|
|
"reward_std": 0.44006140530109406,
|
|
"rewards/reward_fn/mean": -0.225,
|
|
"rewards/reward_fn/std": 0.440061417222023,
|
|
"sampling/importance_sampling_ratio/max": 1.3078218579292298,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001706719398498,
|
|
"sampling/importance_sampling_ratio/min": 0.7689285039901733,
|
|
"sampling/sampling_logp_difference/max": 0.3046458959579468,
|
|
"sampling/sampling_logp_difference/mean": 0.02199532762169838,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 318.3,
|
|
"completions/max_terminated_length": 318.3,
|
|
"completions/mean_length": 219.8375,
|
|
"completions/mean_terminated_length": 219.8375,
|
|
"completions/min_length": 146.6,
|
|
"completions/min_terminated_length": 146.6,
|
|
"entropy": 1.0886065609753133,
|
|
"epoch": 0.21421347798106097,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.624151706695557,
|
|
"kl": 0.05127502836985513,
|
|
"learning_rate": 4.993765532612692e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 10145217.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.4553533583879471,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.455353370308876,
|
|
"sampling/importance_sampling_ratio/max": 1.26222323179245,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998669266700745,
|
|
"sampling/importance_sampling_ratio/min": 0.7888380587100983,
|
|
"sampling/sampling_logp_difference/max": 0.2595196783542633,
|
|
"sampling/sampling_logp_difference/mean": 0.020886969193816184,
|
|
"step": 2330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.3,
|
|
"completions/max_terminated_length": 312.3,
|
|
"completions/mean_length": 211.6375,
|
|
"completions/mean_terminated_length": 211.6375,
|
|
"completions/min_length": 138.9,
|
|
"completions/min_terminated_length": 138.9,
|
|
"entropy": 1.0902759991586208,
|
|
"epoch": 0.21513284913119426,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.470648288726807,
|
|
"kl": 0.06268089526565745,
|
|
"learning_rate": 4.993585432056299e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 10187236.0,
|
|
"reward": -0.2,
|
|
"reward_std": 0.41113159954547884,
|
|
"rewards/reward_fn/mean": -0.2,
|
|
"rewards/reward_fn/std": 0.41113162934780123,
|
|
"sampling/importance_sampling_ratio/max": 1.2463592410087585,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999272048473358,
|
|
"sampling/importance_sampling_ratio/min": 0.771414703130722,
|
|
"sampling/sampling_logp_difference/max": 0.2705600321292877,
|
|
"sampling/sampling_logp_difference/mean": 0.021528172120451928,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 334.9,
|
|
"completions/max_terminated_length": 334.9,
|
|
"completions/mean_length": 239.75,
|
|
"completions/mean_terminated_length": 239.75,
|
|
"completions/min_length": 164.6,
|
|
"completions/min_terminated_length": 164.6,
|
|
"entropy": 1.0869650952517986,
|
|
"epoch": 0.21605222028132756,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.322874069213867,
|
|
"kl": 0.05598060296906624,
|
|
"learning_rate": 4.993402770387782e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 10230712.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.3156497091054916,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.3156497091054916,
|
|
"sampling/importance_sampling_ratio/max": 1.235500407218933,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997797667980194,
|
|
"sampling/importance_sampling_ratio/min": 0.7792201340198517,
|
|
"sampling/sampling_logp_difference/max": 0.254269540309906,
|
|
"sampling/sampling_logp_difference/mean": 0.02162648644298315,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 257.3,
|
|
"completions/max_terminated_length": 257.3,
|
|
"completions/mean_length": 196.925,
|
|
"completions/mean_terminated_length": 196.925,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 1.1058829128742218,
|
|
"epoch": 0.2169715914314609,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.16930444538593292,
|
|
"kl": 0.06482969542848878,
|
|
"learning_rate": 4.993217547794748e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 10274154.0,
|
|
"reward": 0.11875,
|
|
"reward_std": 0.3333532750606537,
|
|
"rewards/reward_fn/mean": 0.11875,
|
|
"rewards/reward_fn/std": 0.3333532869815826,
|
|
"sampling/importance_sampling_ratio/max": 1.2637481212615966,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993589758872986,
|
|
"sampling/importance_sampling_ratio/min": 0.7624344527721405,
|
|
"sampling/sampling_logp_difference/max": 0.2947443604469299,
|
|
"sampling/sampling_logp_difference/mean": 0.021724693663418292,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.6,
|
|
"completions/max_terminated_length": 316.6,
|
|
"completions/mean_length": 204.8625,
|
|
"completions/mean_terminated_length": 204.8625,
|
|
"completions/min_length": 124.9,
|
|
"completions/min_terminated_length": 124.9,
|
|
"entropy": 1.0692421056330204,
|
|
"epoch": 0.21789096258159418,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.15967492759227753,
|
|
"kl": 0.20456191007688176,
|
|
"learning_rate": 4.993029764467438e-06,
|
|
"loss": 0.01,
|
|
"num_tokens": 10317031.0,
|
|
"reward": -0.275,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/reward_fn/mean": -0.275,
|
|
"rewards/reward_fn/std": 0.2504029482603073,
|
|
"sampling/importance_sampling_ratio/max": 1.2576494455337524,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995150744915009,
|
|
"sampling/importance_sampling_ratio/min": 0.795537668466568,
|
|
"sampling/sampling_logp_difference/max": 0.2508530020713806,
|
|
"sampling/sampling_logp_difference/mean": 0.020893950760364533,
|
|
"step": 2370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.7,
|
|
"completions/max_terminated_length": 281.7,
|
|
"completions/mean_length": 199.5125,
|
|
"completions/mean_terminated_length": 199.5125,
|
|
"completions/min_length": 131.4,
|
|
"completions/min_terminated_length": 131.4,
|
|
"entropy": 1.0404366724193097,
|
|
"epoch": 0.2188103337317275,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.369706153869629,
|
|
"kl": 0.05878325866069645,
|
|
"learning_rate": 4.992839420598719e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 10362424.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.3896386116743088,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.3896386355161667,
|
|
"sampling/importance_sampling_ratio/max": 1.327103841304779,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000702798366548,
|
|
"sampling/importance_sampling_ratio/min": 0.7723352551460266,
|
|
"sampling/sampling_logp_difference/max": 0.3010377049446106,
|
|
"sampling/sampling_logp_difference/mean": 0.020696754939854144,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.8,
|
|
"completions/max_terminated_length": 308.8,
|
|
"completions/mean_length": 218.1375,
|
|
"completions/mean_terminated_length": 218.1375,
|
|
"completions/min_length": 145.8,
|
|
"completions/min_terminated_length": 145.8,
|
|
"entropy": 1.0392695881426335,
|
|
"epoch": 0.2197297048818608,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.291379928588867,
|
|
"kl": 0.05432071414543316,
|
|
"learning_rate": 4.992646516384089e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 10406099.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.2478467583656311,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.24784677624702453,
|
|
"sampling/importance_sampling_ratio/max": 1.319090974330902,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999670863151551,
|
|
"sampling/importance_sampling_ratio/min": 0.7603765904903412,
|
|
"sampling/sampling_logp_difference/max": 0.33032592535018923,
|
|
"sampling/sampling_logp_difference/mean": 0.02081177644431591,
|
|
"step": 2390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 313.5,
|
|
"completions/max_terminated_length": 292.0,
|
|
"completions/mean_length": 213.6125,
|
|
"completions/mean_terminated_length": 198.75,
|
|
"completions/min_length": 144.3,
|
|
"completions/min_terminated_length": 144.3,
|
|
"entropy": 1.0136086121201515,
|
|
"epoch": 0.22064907603199413,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.112517356872559,
|
|
"kl": 0.05415805063676089,
|
|
"learning_rate": 4.992451052021676e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 10449860.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.42108623385429383,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.42108623385429383,
|
|
"sampling/importance_sampling_ratio/max": 1.2796338558197022,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999727964401245,
|
|
"sampling/importance_sampling_ratio/min": 0.7964086532592773,
|
|
"sampling/sampling_logp_difference/max": 0.26001877784729005,
|
|
"sampling/sampling_logp_difference/mean": 0.020032955892384054,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"epoch": 0.22064907603199413,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.001963350785340314,
|
|
"eval_completions/max_length": 284.1535776614311,
|
|
"eval_completions/max_terminated_length": 282.23385689354274,
|
|
"eval_completions/mean_length": 202.07482547993018,
|
|
"eval_completions/mean_terminated_length": 201.57536587540392,
|
|
"eval_completions/min_length": 132.80104712041884,
|
|
"eval_completions/min_terminated_length": 132.80104712041884,
|
|
"eval_entropy": 1.0633187783430176,
|
|
"eval_frac_reward_zero_std": 0.3333333333333333,
|
|
"eval_kl": 0.051979973414070894,
|
|
"eval_loss": 0.0027242586947977543,
|
|
"eval_num_tokens": 10449860.0,
|
|
"eval_reward": 0.03861256544502618,
|
|
"eval_reward_std": 0.32740326946110415,
|
|
"eval_rewards/reward_fn/mean": 0.03861256544502618,
|
|
"eval_rewards/reward_fn/std": 0.3274032780949358,
|
|
"eval_runtime": 1668.9665,
|
|
"eval_samples_per_second": 0.343,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2727945083931063,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999052890516285,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.771780295323509,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2922986398920755,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021046556556763137,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.9,
|
|
"completions/max_terminated_length": 314.9,
|
|
"completions/mean_length": 208.475,
|
|
"completions/mean_terminated_length": 208.475,
|
|
"completions/min_length": 136.3,
|
|
"completions/min_terminated_length": 136.3,
|
|
"entropy": 1.005654465407133,
|
|
"epoch": 0.22156844718212743,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.824005603790283,
|
|
"kl": 0.06712795995990746,
|
|
"learning_rate": 4.992253027712238e-06,
|
|
"loss": 0.0036,
|
|
"num_tokens": 10494194.0,
|
|
"reward": -0.15625,
|
|
"reward_std": 0.4667517691850662,
|
|
"rewards/reward_fn/mean": -0.15625,
|
|
"rewards/reward_fn/std": 0.46675178706645964,
|
|
"sampling/importance_sampling_ratio/max": 1.3075680613517762,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999204099178314,
|
|
"sampling/importance_sampling_ratio/min": 0.7279807150363922,
|
|
"sampling/sampling_logp_difference/max": 0.33991732001304625,
|
|
"sampling/sampling_logp_difference/mean": 0.020904942788183688,
|
|
"step": 2410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 252.1,
|
|
"completions/max_terminated_length": 252.1,
|
|
"completions/mean_length": 169.075,
|
|
"completions/mean_terminated_length": 169.075,
|
|
"completions/min_length": 105.1,
|
|
"completions/min_terminated_length": 105.1,
|
|
"entropy": 1.0016811445355416,
|
|
"epoch": 0.22248781833226072,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1650761067867279,
|
|
"kl": 0.04423586474149488,
|
|
"learning_rate": 4.992052443659163e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 10531160.0,
|
|
"reward": 0.08125,
|
|
"reward_std": 0.44778977036476136,
|
|
"rewards/reward_fn/mean": 0.08125,
|
|
"rewards/reward_fn/std": 0.44778978228569033,
|
|
"sampling/importance_sampling_ratio/max": 1.2289716601371765,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003277361392975,
|
|
"sampling/importance_sampling_ratio/min": 0.7768762826919555,
|
|
"sampling/sampling_logp_difference/max": 0.2542852520942688,
|
|
"sampling/sampling_logp_difference/mean": 0.020382095873355866,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 241.8,
|
|
"completions/max_terminated_length": 218.0,
|
|
"completions/mean_length": 164.775,
|
|
"completions/mean_terminated_length": 160.76071472167968,
|
|
"completions/min_length": 105.7,
|
|
"completions/min_terminated_length": 105.7,
|
|
"entropy": 1.0449124954640865,
|
|
"epoch": 0.22340718948239405,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.653650283813477,
|
|
"kl": 0.04148949554655701,
|
|
"learning_rate": 4.9918493000684664e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 10573358.0,
|
|
"reward": 0.16875,
|
|
"reward_std": 0.40078147053718566,
|
|
"rewards/reward_fn/mean": 0.16875,
|
|
"rewards/reward_fn/std": 0.4007814884185791,
|
|
"sampling/importance_sampling_ratio/max": 1.247040319442749,
|
|
"sampling/importance_sampling_ratio/mean": 0.999780535697937,
|
|
"sampling/importance_sampling_ratio/min": 0.774491935968399,
|
|
"sampling/sampling_logp_difference/max": 0.26907032132148745,
|
|
"sampling/sampling_logp_difference/mean": 0.021498653292655944,
|
|
"step": 2430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.4,
|
|
"completions/max_terminated_length": 280.4,
|
|
"completions/mean_length": 200.8125,
|
|
"completions/mean_terminated_length": 200.8125,
|
|
"completions/min_length": 130.7,
|
|
"completions/min_terminated_length": 130.7,
|
|
"entropy": 1.1084297522902489,
|
|
"epoch": 0.22432656063252734,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.192923545837402,
|
|
"kl": 0.04346291564288549,
|
|
"learning_rate": 4.991643597148792e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 10617647.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.5326397061347962,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.5326397299766541,
|
|
"sampling/importance_sampling_ratio/max": 1.2513515949249268,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998542249202729,
|
|
"sampling/importance_sampling_ratio/min": 0.7990241110324859,
|
|
"sampling/sampling_logp_difference/max": 0.24619550704956056,
|
|
"sampling/sampling_logp_difference/mean": 0.022279351390898226,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.2,
|
|
"completions/max_terminated_length": 315.2,
|
|
"completions/mean_length": 210.6125,
|
|
"completions/mean_terminated_length": 210.6125,
|
|
"completions/min_length": 120.6,
|
|
"completions/min_terminated_length": 120.6,
|
|
"entropy": 0.9969607658684254,
|
|
"epoch": 0.22524593178266067,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.2507394552230835,
|
|
"kl": 0.03999340662267059,
|
|
"learning_rate": 4.991435335111413e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 10661576.0,
|
|
"reward": -0.3125,
|
|
"reward_std": 0.42823243141174316,
|
|
"rewards/reward_fn/mean": -0.3125,
|
|
"rewards/reward_fn/std": 0.42823246121406555,
|
|
"sampling/importance_sampling_ratio/max": 1.3170373082160949,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001026749610902,
|
|
"sampling/importance_sampling_ratio/min": 0.7803112447261811,
|
|
"sampling/sampling_logp_difference/max": 0.3029950797557831,
|
|
"sampling/sampling_logp_difference/mean": 0.02015311736613512,
|
|
"step": 2450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.2,
|
|
"completions/max_terminated_length": 258.2,
|
|
"completions/mean_length": 191.875,
|
|
"completions/mean_terminated_length": 191.875,
|
|
"completions/min_length": 132.8,
|
|
"completions/min_terminated_length": 132.8,
|
|
"entropy": 1.0560759492218494,
|
|
"epoch": 0.22616530293279397,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.39122486114502,
|
|
"kl": 0.037859664915595205,
|
|
"learning_rate": 4.991224514170232e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 10702694.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.38138206005096437,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.38138206005096437,
|
|
"sampling/importance_sampling_ratio/max": 1.2704686760902404,
|
|
"sampling/importance_sampling_ratio/mean": 0.999655568599701,
|
|
"sampling/importance_sampling_ratio/min": 0.7703635215759277,
|
|
"sampling/sampling_logp_difference/max": 0.2721349120140076,
|
|
"sampling/sampling_logp_difference/mean": 0.020930757001042365,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.0,
|
|
"completions/max_terminated_length": 288.0,
|
|
"completions/mean_length": 189.975,
|
|
"completions/mean_terminated_length": 189.975,
|
|
"completions/min_length": 110.1,
|
|
"completions/min_terminated_length": 110.1,
|
|
"entropy": 1.0435327395796776,
|
|
"epoch": 0.2270846740829273,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.204001426696777,
|
|
"kl": 0.046926383359823375,
|
|
"learning_rate": 4.9910111345417785e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 10744932.0,
|
|
"reward": 0.13125,
|
|
"reward_std": 0.2790661096572876,
|
|
"rewards/reward_fn/mean": 0.13125,
|
|
"rewards/reward_fn/std": 0.2790661096572876,
|
|
"sampling/importance_sampling_ratio/max": 1.2310782432556153,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001035153865814,
|
|
"sampling/importance_sampling_ratio/min": 0.7943857610225677,
|
|
"sampling/sampling_logp_difference/max": 0.2332480013370514,
|
|
"sampling/sampling_logp_difference/mean": 0.020773524418473244,
|
|
"step": 2470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 262.5,
|
|
"completions/max_terminated_length": 262.5,
|
|
"completions/mean_length": 183.1875,
|
|
"completions/mean_terminated_length": 183.1875,
|
|
"completions/min_length": 120.3,
|
|
"completions/min_terminated_length": 120.3,
|
|
"entropy": 1.022875565290451,
|
|
"epoch": 0.2280040452330606,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.991796970367432,
|
|
"kl": 0.0434352322074119,
|
|
"learning_rate": 4.990795196445212e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 10783203.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.45646098256111145,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.45646100044250487,
|
|
"sampling/importance_sampling_ratio/max": 1.2591790556907654,
|
|
"sampling/importance_sampling_ratio/mean": 1.000213861465454,
|
|
"sampling/importance_sampling_ratio/min": 0.7987667083740234,
|
|
"sampling/sampling_logp_difference/max": 0.2563947081565857,
|
|
"sampling/sampling_logp_difference/mean": 0.02045534607023001,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.0,
|
|
"completions/max_terminated_length": 287.0,
|
|
"completions/mean_length": 187.45,
|
|
"completions/mean_terminated_length": 187.45,
|
|
"completions/min_length": 113.9,
|
|
"completions/min_terminated_length": 113.9,
|
|
"entropy": 1.0187353186309338,
|
|
"epoch": 0.22892341638319388,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.596317768096924,
|
|
"kl": 0.031543022918049245,
|
|
"learning_rate": 4.990576700102316e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 10822711.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.4136129081249237,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.41361292004585265,
|
|
"sampling/importance_sampling_ratio/max": 1.2634188771247863,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004124641418457,
|
|
"sampling/importance_sampling_ratio/min": 0.7752062618732453,
|
|
"sampling/sampling_logp_difference/max": 0.2689652442932129,
|
|
"sampling/sampling_logp_difference/mean": 0.020628484524786474,
|
|
"step": 2490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.7,
|
|
"completions/max_terminated_length": 273.7,
|
|
"completions/mean_length": 188.8375,
|
|
"completions/mean_terminated_length": 188.8375,
|
|
"completions/min_length": 125.1,
|
|
"completions/min_terminated_length": 125.1,
|
|
"entropy": 0.9977653473615646,
|
|
"epoch": 0.2298427875333272,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.077992916107178,
|
|
"kl": 0.04374045781441964,
|
|
"learning_rate": 4.990355645737506e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 10863234.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.4556564152240753,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.45565642714500426,
|
|
"sampling/importance_sampling_ratio/max": 1.2421785593032837,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000094532966615,
|
|
"sampling/importance_sampling_ratio/min": 0.798731905221939,
|
|
"sampling/sampling_logp_difference/max": 0.24611550569534302,
|
|
"sampling/sampling_logp_difference/mean": 0.019924255460500716,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.4,
|
|
"completions/max_terminated_length": 291.4,
|
|
"completions/mean_length": 209.8125,
|
|
"completions/mean_terminated_length": 209.8125,
|
|
"completions/min_length": 149.2,
|
|
"completions/min_terminated_length": 149.2,
|
|
"entropy": 1.073886164277792,
|
|
"epoch": 0.2307621586834605,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.000349521636963,
|
|
"kl": 0.03493021497270092,
|
|
"learning_rate": 4.99013203357782e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 10906875.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.4305500477552414,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.43055006563663484,
|
|
"sampling/importance_sampling_ratio/max": 1.2677898645401,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999549925327301,
|
|
"sampling/importance_sampling_ratio/min": 0.7775612354278565,
|
|
"sampling/sampling_logp_difference/max": 0.27662190794944763,
|
|
"sampling/sampling_logp_difference/mean": 0.02086197044700384,
|
|
"step": 2510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.4,
|
|
"completions/max_terminated_length": 313.4,
|
|
"completions/mean_length": 218.3125,
|
|
"completions/mean_terminated_length": 218.3125,
|
|
"completions/min_length": 142.9,
|
|
"completions/min_terminated_length": 142.9,
|
|
"entropy": 1.1404215224087237,
|
|
"epoch": 0.23168152983359383,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.994783878326416,
|
|
"kl": 0.030836883222218604,
|
|
"learning_rate": 4.989905863852928e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 10948908.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.3935216277837753,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.3935216277837753,
|
|
"sampling/importance_sampling_ratio/max": 1.2588043093681336,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003569304943085,
|
|
"sampling/importance_sampling_ratio/min": 0.7848890483379364,
|
|
"sampling/sampling_logp_difference/max": 0.2565630555152893,
|
|
"sampling/sampling_logp_difference/mean": 0.021893820166587828,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 265.7,
|
|
"completions/max_terminated_length": 265.7,
|
|
"completions/mean_length": 200.6875,
|
|
"completions/mean_terminated_length": 200.6875,
|
|
"completions/min_length": 142.3,
|
|
"completions/min_terminated_length": 142.3,
|
|
"entropy": 1.1183612763881683,
|
|
"epoch": 0.23260090098372713,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.919711589813232,
|
|
"kl": 0.03401488094823435,
|
|
"learning_rate": 4.989677136795123e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 10989299.0,
|
|
"reward": -0.04375,
|
|
"reward_std": 0.44345162659883497,
|
|
"rewards/reward_fn/mean": -0.04375,
|
|
"rewards/reward_fn/std": 0.4434516325592995,
|
|
"sampling/importance_sampling_ratio/max": 1.2850476384162903,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999486744403839,
|
|
"sampling/importance_sampling_ratio/min": 0.7547445714473724,
|
|
"sampling/sampling_logp_difference/max": 0.3379413396120071,
|
|
"sampling/sampling_logp_difference/mean": 0.02173040695488453,
|
|
"step": 2530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.8,
|
|
"completions/max_terminated_length": 315.8,
|
|
"completions/mean_length": 218.45,
|
|
"completions/mean_terminated_length": 218.45,
|
|
"completions/min_length": 149.4,
|
|
"completions/min_terminated_length": 149.4,
|
|
"entropy": 1.1781387448310852,
|
|
"epoch": 0.23352027213386045,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.417177200317383,
|
|
"kl": 0.03935691156657413,
|
|
"learning_rate": 4.989445852639328e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 11036847.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.5068073570728302,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.5068073749542237,
|
|
"sampling/importance_sampling_ratio/max": 1.3013556718826294,
|
|
"sampling/importance_sampling_ratio/mean": 0.999855923652649,
|
|
"sampling/importance_sampling_ratio/min": 0.7862894415855408,
|
|
"sampling/sampling_logp_difference/max": 0.27718343734741213,
|
|
"sampling/sampling_logp_difference/mean": 0.021141526848077775,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.3,
|
|
"completions/max_terminated_length": 288.3,
|
|
"completions/mean_length": 195.7875,
|
|
"completions/mean_terminated_length": 195.7875,
|
|
"completions/min_length": 112.5,
|
|
"completions/min_terminated_length": 112.5,
|
|
"entropy": 1.145099437236786,
|
|
"epoch": 0.23443964328399375,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.554821491241455,
|
|
"kl": 0.050792729353997854,
|
|
"learning_rate": 4.989212011623087e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 11080366.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.3682431042194366,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.36824311017990113,
|
|
"sampling/importance_sampling_ratio/max": 1.3067343711853028,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003229796886444,
|
|
"sampling/importance_sampling_ratio/min": 0.7865084171295166,
|
|
"sampling/sampling_logp_difference/max": 0.2849071264266968,
|
|
"sampling/sampling_logp_difference/mean": 0.02231056559830904,
|
|
"step": 2550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.7,
|
|
"completions/max_terminated_length": 304.7,
|
|
"completions/mean_length": 204.5875,
|
|
"completions/mean_terminated_length": 204.5875,
|
|
"completions/min_length": 132.1,
|
|
"completions/min_terminated_length": 132.1,
|
|
"entropy": 1.118402624130249,
|
|
"epoch": 0.23535901443412705,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.418643474578857,
|
|
"kl": 0.033206678985152394,
|
|
"learning_rate": 4.988975613986575e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 11124397.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.4032860606908798,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.4032860606908798,
|
|
"sampling/importance_sampling_ratio/max": 1.2581639528274535,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003222167491912,
|
|
"sampling/importance_sampling_ratio/min": 0.7984795153141022,
|
|
"sampling/sampling_logp_difference/max": 0.24500706195831298,
|
|
"sampling/sampling_logp_difference/mean": 0.02116144709289074,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.1,
|
|
"completions/max_terminated_length": 294.1,
|
|
"completions/mean_length": 216.1625,
|
|
"completions/mean_terminated_length": 216.1625,
|
|
"completions/min_length": 145.0,
|
|
"completions/min_terminated_length": 145.0,
|
|
"entropy": 1.0753696747124195,
|
|
"epoch": 0.23627838558426037,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.658927917480469,
|
|
"kl": 0.0454733666498214,
|
|
"learning_rate": 4.988736659972593e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 11170010.0,
|
|
"reward": -0.16875,
|
|
"reward_std": 0.4471833348274231,
|
|
"rewards/reward_fn/mean": -0.16875,
|
|
"rewards/reward_fn/std": 0.44718335270881654,
|
|
"sampling/importance_sampling_ratio/max": 1.2791348099708557,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001384019851685,
|
|
"sampling/importance_sampling_ratio/min": 0.7986905455589295,
|
|
"sampling/sampling_logp_difference/max": 0.24947422742843628,
|
|
"sampling/sampling_logp_difference/mean": 0.020509620755910875,
|
|
"step": 2570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.7,
|
|
"completions/max_terminated_length": 278.7,
|
|
"completions/mean_length": 200.8125,
|
|
"completions/mean_terminated_length": 200.8125,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 1.018815215677023,
|
|
"epoch": 0.23719775673439367,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1251971423625946,
|
|
"kl": 0.03866477177361958,
|
|
"learning_rate": 4.988495149826563e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 11215067.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.40160019397735597,
|
|
"rewards/reward_fn/mean": 0.3375,
|
|
"rewards/reward_fn/std": 0.40160020589828493,
|
|
"sampling/importance_sampling_ratio/max": 1.300172507762909,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998606741428375,
|
|
"sampling/importance_sampling_ratio/min": 0.7573921799659729,
|
|
"sampling/sampling_logp_difference/max": 0.3145493656396866,
|
|
"sampling/sampling_logp_difference/mean": 0.02019682712852955,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.2,
|
|
"completions/max_terminated_length": 284.2,
|
|
"completions/mean_length": 189.3375,
|
|
"completions/mean_terminated_length": 189.3375,
|
|
"completions/min_length": 121.9,
|
|
"completions/min_terminated_length": 121.9,
|
|
"entropy": 1.0066481456160545,
|
|
"epoch": 0.238117127884527,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.907307147979736,
|
|
"kl": 0.04344964168849401,
|
|
"learning_rate": 4.988251083796539e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 11257534.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.4938108593225479,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4938108831644058,
|
|
"sampling/importance_sampling_ratio/max": 1.2580394625663758,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001824855804444,
|
|
"sampling/importance_sampling_ratio/min": 0.7979860544204712,
|
|
"sampling/sampling_logp_difference/max": 0.24451079964637756,
|
|
"sampling/sampling_logp_difference/mean": 0.020432468689978122,
|
|
"step": 2590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.8,
|
|
"completions/max_terminated_length": 279.8,
|
|
"completions/mean_length": 197.4875,
|
|
"completions/mean_terminated_length": 197.4875,
|
|
"completions/min_length": 144.9,
|
|
"completions/min_terminated_length": 144.9,
|
|
"entropy": 1.098967546224594,
|
|
"epoch": 0.2390364990346603,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 7.8013458251953125,
|
|
"kl": 0.038584265240933745,
|
|
"learning_rate": 4.988004462133192e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 11300565.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.2140781342983246,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.21407814025878907,
|
|
"sampling/importance_sampling_ratio/max": 1.2968656063079833,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996149718761445,
|
|
"sampling/importance_sampling_ratio/min": 0.7863863348960877,
|
|
"sampling/sampling_logp_difference/max": 0.2896435022354126,
|
|
"sampling/sampling_logp_difference/mean": 0.02172896321862936,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"epoch": 0.2390364990346603,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0010907504363001745,
|
|
"eval_completions/max_length": 280.4066317626527,
|
|
"eval_completions/max_terminated_length": 279.44153577661433,
|
|
"eval_completions/mean_length": 198.2931937172775,
|
|
"eval_completions/mean_terminated_length": 197.99836912038646,
|
|
"eval_completions/min_length": 126.51657940663176,
|
|
"eval_completions/min_terminated_length": 126.51657940663176,
|
|
"eval_entropy": 1.105177756067346,
|
|
"eval_frac_reward_zero_std": 0.32460732984293195,
|
|
"eval_kl": 0.048586749424589036,
|
|
"eval_loss": 0.0025696163065731525,
|
|
"eval_num_tokens": 11300565.0,
|
|
"eval_reward": 0.06140924956369982,
|
|
"eval_reward_std": 0.33572474153254045,
|
|
"eval_rewards/reward_fn/mean": 0.06140924956369982,
|
|
"eval_rewards/reward_fn/std": 0.3357247505824603,
|
|
"eval_runtime": 1649.9815,
|
|
"eval_samples_per_second": 0.347,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2648652752448544,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999690453211466,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7784229363030372,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27980934148475556,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021116749835355015,
|
|
"eval_steps_per_second": 0.044,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.2,
|
|
"completions/max_terminated_length": 268.2,
|
|
"completions/mean_length": 185.9625,
|
|
"completions/mean_terminated_length": 185.9625,
|
|
"completions/min_length": 126.4,
|
|
"completions/min_terminated_length": 126.4,
|
|
"entropy": 0.964335186779499,
|
|
"epoch": 0.2399558701847936,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.13531634211540222,
|
|
"kl": 0.04779042224399745,
|
|
"learning_rate": 4.9877552850898255e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 11337506.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.15235702097415924,
|
|
"sampling/importance_sampling_ratio/max": 1.2524919152259826,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994821786880493,
|
|
"sampling/importance_sampling_ratio/min": 0.7655371785163879,
|
|
"sampling/sampling_logp_difference/max": 0.28164924383163453,
|
|
"sampling/sampling_logp_difference/mean": 0.020220484212040903,
|
|
"step": 2610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.4,
|
|
"completions/max_terminated_length": 294.4,
|
|
"completions/mean_length": 211.9625,
|
|
"completions/mean_terminated_length": 211.9625,
|
|
"completions/min_length": 137.6,
|
|
"completions/min_terminated_length": 137.6,
|
|
"entropy": 1.0349913381040097,
|
|
"epoch": 0.2408752413349269,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.048066139221191,
|
|
"kl": 0.04933973160805181,
|
|
"learning_rate": 4.987503552922363e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 11380551.0,
|
|
"reward": 0.11875,
|
|
"reward_std": 0.4711076617240906,
|
|
"rewards/reward_fn/mean": 0.11875,
|
|
"rewards/reward_fn/std": 0.47110766768455503,
|
|
"sampling/importance_sampling_ratio/max": 1.2970452785491944,
|
|
"sampling/importance_sampling_ratio/mean": 1.000468409061432,
|
|
"sampling/importance_sampling_ratio/min": 0.7365161001682281,
|
|
"sampling/sampling_logp_difference/max": 0.33642680644989015,
|
|
"sampling/sampling_logp_difference/mean": 0.020787146501243116,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 256.5,
|
|
"completions/max_terminated_length": 256.5,
|
|
"completions/mean_length": 195.4625,
|
|
"completions/mean_terminated_length": 195.4625,
|
|
"completions/min_length": 135.7,
|
|
"completions/min_terminated_length": 135.7,
|
|
"entropy": 1.0658831246197225,
|
|
"epoch": 0.2417946124850602,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.394301414489746,
|
|
"kl": 0.04424796831444837,
|
|
"learning_rate": 4.987249265889355e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 11423716.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.34450737535953524,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.34450739324092866,
|
|
"sampling/importance_sampling_ratio/max": 1.2958876371383667,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003436505794525,
|
|
"sampling/importance_sampling_ratio/min": 0.7792130589485169,
|
|
"sampling/sampling_logp_difference/max": 0.27105374336242677,
|
|
"sampling/sampling_logp_difference/mean": 0.021434491500258446,
|
|
"step": 2630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.0,
|
|
"completions/max_terminated_length": 289.0,
|
|
"completions/mean_length": 203.8875,
|
|
"completions/mean_terminated_length": 203.8875,
|
|
"completions/min_length": 132.4,
|
|
"completions/min_terminated_length": 132.4,
|
|
"entropy": 1.0949511714279652,
|
|
"epoch": 0.24271398363519353,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.1297372430562973,
|
|
"kl": 0.03642780394293368,
|
|
"learning_rate": 4.9869924242519715e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 11466379.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.24982124865055083,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.24982124865055083,
|
|
"sampling/importance_sampling_ratio/max": 1.2476243257522583,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994765996932984,
|
|
"sampling/importance_sampling_ratio/min": 0.7869486212730408,
|
|
"sampling/sampling_logp_difference/max": 0.2617160797119141,
|
|
"sampling/sampling_logp_difference/mean": 0.021958728693425654,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.9,
|
|
"completions/max_terminated_length": 292.9,
|
|
"completions/mean_length": 197.0875,
|
|
"completions/mean_terminated_length": 197.0875,
|
|
"completions/min_length": 125.9,
|
|
"completions/min_terminated_length": 125.9,
|
|
"entropy": 1.151274362206459,
|
|
"epoch": 0.24363335478532683,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.6219801902771,
|
|
"kl": 0.038672633573878554,
|
|
"learning_rate": 4.986733028274013e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 11509154.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.46739681661128996,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.4673968285322189,
|
|
"sampling/importance_sampling_ratio/max": 1.2946714997291564,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000390529632568,
|
|
"sampling/importance_sampling_ratio/min": 0.8016185164451599,
|
|
"sampling/sampling_logp_difference/max": 0.2692372381687164,
|
|
"sampling/sampling_logp_difference/mean": 0.022010532766580583,
|
|
"step": 2650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.4,
|
|
"completions/max_terminated_length": 281.4,
|
|
"completions/mean_length": 197.125,
|
|
"completions/mean_terminated_length": 197.125,
|
|
"completions/min_length": 134.8,
|
|
"completions/min_terminated_length": 134.8,
|
|
"entropy": 1.1257120065391064,
|
|
"epoch": 0.24455272593546015,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 7.526282787322998,
|
|
"kl": 0.0321127021801658,
|
|
"learning_rate": 4.986471078221898e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 11555284.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.2658330500125885,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.26583306193351747,
|
|
"sampling/importance_sampling_ratio/max": 1.226505410671234,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999152839183807,
|
|
"sampling/importance_sampling_ratio/min": 0.77623330950737,
|
|
"sampling/sampling_logp_difference/max": 0.2550153940916061,
|
|
"sampling/sampling_logp_difference/mean": 0.02201693318784237,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.9,
|
|
"completions/max_terminated_length": 280.9,
|
|
"completions/mean_length": 198.05,
|
|
"completions/mean_terminated_length": 198.05,
|
|
"completions/min_length": 145.0,
|
|
"completions/min_terminated_length": 145.0,
|
|
"entropy": 1.0493835730478167,
|
|
"epoch": 0.24547209708559345,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.10485733300447464,
|
|
"kl": 0.0424958658462856,
|
|
"learning_rate": 4.986206574364671e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 11600976.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.5227612406015396,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.5227612584829331,
|
|
"sampling/importance_sampling_ratio/max": 1.309563684463501,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002075254917144,
|
|
"sampling/importance_sampling_ratio/min": 0.7929507851600647,
|
|
"sampling/sampling_logp_difference/max": 0.28748188018798826,
|
|
"sampling/sampling_logp_difference/mean": 0.02090072529390454,
|
|
"step": 2670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.2,
|
|
"completions/max_terminated_length": 271.2,
|
|
"completions/mean_length": 192.9125,
|
|
"completions/mean_terminated_length": 192.9125,
|
|
"completions/min_length": 123.4,
|
|
"completions/min_terminated_length": 123.4,
|
|
"entropy": 1.05034648925066,
|
|
"epoch": 0.24639146823572677,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.577430248260498,
|
|
"kl": 0.03439225670299493,
|
|
"learning_rate": 4.985939516973996e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 11641457.0,
|
|
"reward": 0.13125,
|
|
"reward_std": 0.46862156093120577,
|
|
"rewards/reward_fn/mean": 0.13125,
|
|
"rewards/reward_fn/std": 0.46862159073352816,
|
|
"sampling/importance_sampling_ratio/max": 1.2239811897277832,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003397941589356,
|
|
"sampling/importance_sampling_ratio/min": 0.7566246837377548,
|
|
"sampling/sampling_logp_difference/max": 0.29382072687149047,
|
|
"sampling/sampling_logp_difference/mean": 0.021501031704246998,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 240.6,
|
|
"completions/max_terminated_length": 240.6,
|
|
"completions/mean_length": 174.7125,
|
|
"completions/mean_terminated_length": 174.7125,
|
|
"completions/min_length": 107.1,
|
|
"completions/min_terminated_length": 107.1,
|
|
"entropy": 0.9959322221577167,
|
|
"epoch": 0.24731083938586007,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 9.488479614257812,
|
|
"kl": 0.030505432945210485,
|
|
"learning_rate": 4.985669906324167e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 11681402.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.28032026886940004,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.280320280790329,
|
|
"sampling/importance_sampling_ratio/max": 1.275475299358368,
|
|
"sampling/importance_sampling_ratio/mean": 1.0008409023284912,
|
|
"sampling/importance_sampling_ratio/min": 0.7753716886043549,
|
|
"sampling/sampling_logp_difference/max": 0.27277711033821106,
|
|
"sampling/sampling_logp_difference/mean": 0.020704777911305428,
|
|
"step": 2690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.6,
|
|
"completions/max_terminated_length": 293.6,
|
|
"completions/mean_length": 214.4375,
|
|
"completions/mean_terminated_length": 214.4375,
|
|
"completions/min_length": 152.3,
|
|
"completions/min_terminated_length": 152.3,
|
|
"entropy": 1.0832776449620725,
|
|
"epoch": 0.24823021053599337,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1169515922665596,
|
|
"kl": 0.04267606807989068,
|
|
"learning_rate": 4.985397742692093e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 11725829.0,
|
|
"reward": -0.45,
|
|
"reward_std": 0.31817767322063445,
|
|
"rewards/reward_fn/mean": -0.45,
|
|
"rewards/reward_fn/std": 0.31817767918109896,
|
|
"sampling/importance_sampling_ratio/max": 1.30653977394104,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001599073410035,
|
|
"sampling/importance_sampling_ratio/min": 0.7692269086837769,
|
|
"sampling/sampling_logp_difference/max": 0.3095731496810913,
|
|
"sampling/sampling_logp_difference/mean": 0.021753770671784876,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 259.4,
|
|
"completions/max_terminated_length": 259.4,
|
|
"completions/mean_length": 197.3625,
|
|
"completions/mean_terminated_length": 197.3625,
|
|
"completions/min_length": 130.7,
|
|
"completions/min_terminated_length": 130.7,
|
|
"entropy": 1.0232362508773805,
|
|
"epoch": 0.2491495816861267,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.008146286010742,
|
|
"kl": 0.05452837008051574,
|
|
"learning_rate": 4.985123026357308e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 11766626.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.408385643362999,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.40838564932346344,
|
|
"sampling/importance_sampling_ratio/max": 1.2262384057044984,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001239717006682,
|
|
"sampling/importance_sampling_ratio/min": 0.7899364292621612,
|
|
"sampling/sampling_logp_difference/max": 0.24692018032073976,
|
|
"sampling/sampling_logp_difference/mean": 0.020400654152035715,
|
|
"step": 2710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.1,
|
|
"completions/max_terminated_length": 258.1,
|
|
"completions/mean_length": 202.7625,
|
|
"completions/mean_terminated_length": 202.7625,
|
|
"completions/min_length": 139.5,
|
|
"completions/min_terminated_length": 139.5,
|
|
"entropy": 0.9118117436766624,
|
|
"epoch": 0.25006895283626,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.27532434463501,
|
|
"kl": 0.039663255779305476,
|
|
"learning_rate": 4.9848457576019685e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 11809999.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.439727446436882,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.4397274643182755,
|
|
"sampling/importance_sampling_ratio/max": 1.238251769542694,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999612867832184,
|
|
"sampling/importance_sampling_ratio/min": 0.7949703752994537,
|
|
"sampling/sampling_logp_difference/max": 0.25368083715438844,
|
|
"sampling/sampling_logp_difference/mean": 0.0191750209312886,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.1,
|
|
"completions/max_terminated_length": 300.1,
|
|
"completions/mean_length": 207.4125,
|
|
"completions/mean_terminated_length": 207.4125,
|
|
"completions/min_length": 134.9,
|
|
"completions/min_terminated_length": 134.9,
|
|
"entropy": 1.0888357855379582,
|
|
"epoch": 0.2509883239863933,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.360905647277832,
|
|
"kl": 0.03997602123417891,
|
|
"learning_rate": 4.98456593671085e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 11852880.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.3181816428899765,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.31818165481090543,
|
|
"sampling/importance_sampling_ratio/max": 1.2487877368927003,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000269055366515,
|
|
"sampling/importance_sampling_ratio/min": 0.7777351140975952,
|
|
"sampling/sampling_logp_difference/max": 0.2791467189788818,
|
|
"sampling/sampling_logp_difference/mean": 0.021814374439418315,
|
|
"step": 2730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 320.3,
|
|
"completions/max_terminated_length": 320.3,
|
|
"completions/mean_length": 226.6,
|
|
"completions/mean_terminated_length": 226.6,
|
|
"completions/min_length": 147.8,
|
|
"completions/min_terminated_length": 147.8,
|
|
"entropy": 1.1566815495491027,
|
|
"epoch": 0.2519076951365266,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.370908737182617,
|
|
"kl": 0.049444525118451564,
|
|
"learning_rate": 4.9842835639713525e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 11895296.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.3311904340982437,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.3311904460191727,
|
|
"sampling/importance_sampling_ratio/max": 1.2461877465248108,
|
|
"sampling/importance_sampling_ratio/mean": 1.000012892484665,
|
|
"sampling/importance_sampling_ratio/min": 0.7453417599201202,
|
|
"sampling/sampling_logp_difference/max": 0.30544832944869993,
|
|
"sampling/sampling_logp_difference/mean": 0.02173358667641878,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.5,
|
|
"completions/max_terminated_length": 297.5,
|
|
"completions/mean_length": 215.5125,
|
|
"completions/mean_terminated_length": 215.5125,
|
|
"completions/min_length": 141.2,
|
|
"completions/min_terminated_length": 141.2,
|
|
"entropy": 1.1005959428846837,
|
|
"epoch": 0.2528270662866599,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.106991291046143,
|
|
"kl": 0.030825991154415533,
|
|
"learning_rate": 4.983998639673496e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 11942673.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.4530158966779709,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.4530159145593643,
|
|
"sampling/importance_sampling_ratio/max": 1.277392816543579,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000280022621155,
|
|
"sampling/importance_sampling_ratio/min": 0.7524430990219116,
|
|
"sampling/sampling_logp_difference/max": 0.3119939923286438,
|
|
"sampling/sampling_logp_difference/mean": 0.02174394391477108,
|
|
"step": 2750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.1,
|
|
"completions/max_terminated_length": 295.1,
|
|
"completions/mean_length": 201.2875,
|
|
"completions/mean_terminated_length": 201.2875,
|
|
"completions/min_length": 133.5,
|
|
"completions/min_terminated_length": 133.5,
|
|
"entropy": 1.0241153724491596,
|
|
"epoch": 0.25374643743679326,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.417367458343506,
|
|
"kl": 0.02174271009862423,
|
|
"learning_rate": 4.983711164109919e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 11987288.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.38901832699775696,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.38901835680007935,
|
|
"sampling/importance_sampling_ratio/max": 1.2437778472900392,
|
|
"sampling/importance_sampling_ratio/mean": 0.999558013677597,
|
|
"sampling/importance_sampling_ratio/min": 0.7623007476329804,
|
|
"sampling/sampling_logp_difference/max": 0.2828593969345093,
|
|
"sampling/sampling_logp_difference/mean": 0.020254597812891007,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.8,
|
|
"completions/max_terminated_length": 305.8,
|
|
"completions/mean_length": 218.3,
|
|
"completions/mean_terminated_length": 218.3,
|
|
"completions/min_length": 147.6,
|
|
"completions/min_terminated_length": 147.6,
|
|
"entropy": 1.0376436173915864,
|
|
"epoch": 0.25466580858692656,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.323155879974365,
|
|
"kl": 0.032405230321455744,
|
|
"learning_rate": 4.983421137575882e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 12032168.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.36794218719005584,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.36794219315052035,
|
|
"sampling/importance_sampling_ratio/max": 1.2814663410186768,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002431273460388,
|
|
"sampling/importance_sampling_ratio/min": 0.7896285057067871,
|
|
"sampling/sampling_logp_difference/max": 0.26454647183418273,
|
|
"sampling/sampling_logp_difference/mean": 0.020564932003617288,
|
|
"step": 2770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 264.6,
|
|
"completions/max_terminated_length": 264.6,
|
|
"completions/mean_length": 185.6875,
|
|
"completions/mean_terminated_length": 185.6875,
|
|
"completions/min_length": 120.4,
|
|
"completions/min_terminated_length": 120.4,
|
|
"entropy": 1.0407466940581798,
|
|
"epoch": 0.25558517973705985,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.12284121662378311,
|
|
"kl": 0.030147026717895643,
|
|
"learning_rate": 4.983128560369267e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 12071335.0,
|
|
"reward": 0.5875,
|
|
"reward_std": 0.29839127957820893,
|
|
"rewards/reward_fn/mean": 0.5875,
|
|
"rewards/reward_fn/std": 0.29839127957820893,
|
|
"sampling/importance_sampling_ratio/max": 1.2712011456489563,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999915659427643,
|
|
"sampling/importance_sampling_ratio/min": 0.8019644439220428,
|
|
"sampling/sampling_logp_difference/max": 0.2486160159111023,
|
|
"sampling/sampling_logp_difference/mean": 0.020536273159086703,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 262.5,
|
|
"completions/max_terminated_length": 262.5,
|
|
"completions/mean_length": 182.2,
|
|
"completions/mean_terminated_length": 182.2,
|
|
"completions/min_length": 119.1,
|
|
"completions/min_terminated_length": 119.1,
|
|
"entropy": 0.982857346534729,
|
|
"epoch": 0.25650455088719315,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.815053939819336,
|
|
"kl": 0.04034502828726545,
|
|
"learning_rate": 4.982833432790573e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 12108215.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.4472267836332321,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.44722679555416106,
|
|
"sampling/importance_sampling_ratio/max": 1.2445849061012269,
|
|
"sampling/importance_sampling_ratio/mean": 1.000073504447937,
|
|
"sampling/importance_sampling_ratio/min": 0.7712437093257904,
|
|
"sampling/sampling_logp_difference/max": 0.2701414227485657,
|
|
"sampling/sampling_logp_difference/mean": 0.020210597664117813,
|
|
"step": 2790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.7,
|
|
"completions/max_terminated_length": 291.7,
|
|
"completions/mean_length": 220.0875,
|
|
"completions/mean_terminated_length": 220.0875,
|
|
"completions/min_length": 149.8,
|
|
"completions/min_terminated_length": 149.8,
|
|
"entropy": 1.1246454559266568,
|
|
"epoch": 0.25742392203732645,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.9768147468566895,
|
|
"kl": 0.031025142641738057,
|
|
"learning_rate": 4.982535755142921e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 12151054.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.5694230645895004,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.5694230824708939,
|
|
"sampling/importance_sampling_ratio/max": 1.3540500164031983,
|
|
"sampling/importance_sampling_ratio/mean": 1.00035600066185,
|
|
"sampling/importance_sampling_ratio/min": 0.7829943418502807,
|
|
"sampling/sampling_logp_difference/max": 0.3097230434417725,
|
|
"sampling/sampling_logp_difference/mean": 0.02193455081433058,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"epoch": 0.25742392203732645,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0017452006980802793,
|
|
"eval_completions/max_length": 288.56893542757416,
|
|
"eval_completions/max_terminated_length": 286.413612565445,
|
|
"eval_completions/mean_length": 204.54886561954623,
|
|
"eval_completions/mean_terminated_length": 204.09514478114264,
|
|
"eval_completions/min_length": 137.4502617801047,
|
|
"eval_completions/min_terminated_length": 137.4502617801047,
|
|
"eval_entropy": 1.1007631256734305,
|
|
"eval_frac_reward_zero_std": 0.27399650959860383,
|
|
"eval_kl": 0.041012291155306926,
|
|
"eval_loss": 0.0021502634044736624,
|
|
"eval_num_tokens": 12151054.0,
|
|
"eval_reward": 0.052683246073298426,
|
|
"eval_reward_std": 0.38608684082617933,
|
|
"eval_rewards/reward_fn/mean": 0.052683246073298426,
|
|
"eval_rewards/reward_fn/std": 0.3860868508122978,
|
|
"eval_runtime": 1689.5098,
|
|
"eval_samples_per_second": 0.339,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2644031249623857,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.000042509034042,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7802122043897552,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27356586354357204,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021417336515679203,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 239.4,
|
|
"completions/max_terminated_length": 239.4,
|
|
"completions/mean_length": 179.15,
|
|
"completions/mean_terminated_length": 179.15,
|
|
"completions/min_length": 120.7,
|
|
"completions/min_terminated_length": 120.7,
|
|
"entropy": 1.0156222365796566,
|
|
"epoch": 0.2583432931874598,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.617149353027344,
|
|
"kl": 0.04966756717767566,
|
|
"learning_rate": 4.9822355277320475e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 12191274.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.30269542038440705,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.302695432305336,
|
|
"sampling/importance_sampling_ratio/max": 1.2694220304489137,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000813603401184,
|
|
"sampling/importance_sampling_ratio/min": 0.7904092967510223,
|
|
"sampling/sampling_logp_difference/max": 0.2642872095108032,
|
|
"sampling/sampling_logp_difference/mean": 0.02089224271476269,
|
|
"step": 2810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.5,
|
|
"completions/max_terminated_length": 278.5,
|
|
"completions/mean_length": 192.5875,
|
|
"completions/mean_terminated_length": 192.5875,
|
|
"completions/min_length": 128.0,
|
|
"completions/min_terminated_length": 128.0,
|
|
"entropy": 1.040990813449025,
|
|
"epoch": 0.2592626643375931,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 2.842435359954834,
|
|
"kl": 0.04421546008961741,
|
|
"learning_rate": 4.981932750866313e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 12233257.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.45548935830593107,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.45548937022686004,
|
|
"sampling/importance_sampling_ratio/max": 1.2264577865600585,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000287413597106,
|
|
"sampling/importance_sampling_ratio/min": 0.7567507982254028,
|
|
"sampling/sampling_logp_difference/max": 0.28429675102233887,
|
|
"sampling/sampling_logp_difference/mean": 0.021971827000379564,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.1,
|
|
"completions/max_terminated_length": 292.1,
|
|
"completions/mean_length": 206.8625,
|
|
"completions/mean_terminated_length": 206.8625,
|
|
"completions/min_length": 145.2,
|
|
"completions/min_terminated_length": 145.2,
|
|
"entropy": 1.071357686072588,
|
|
"epoch": 0.2601820354877264,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.80636739730835,
|
|
"kl": 0.03778555438038893,
|
|
"learning_rate": 4.981627424856691e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 12276054.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.3947398692369461,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.39473988115787506,
|
|
"sampling/importance_sampling_ratio/max": 1.2608086943626404,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997180044651032,
|
|
"sampling/importance_sampling_ratio/min": 0.7502443373203278,
|
|
"sampling/sampling_logp_difference/max": 0.3030838966369629,
|
|
"sampling/sampling_logp_difference/mean": 0.021685385145246983,
|
|
"step": 2830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 320.5,
|
|
"completions/max_terminated_length": 320.5,
|
|
"completions/mean_length": 218.575,
|
|
"completions/mean_terminated_length": 218.575,
|
|
"completions/min_length": 152.7,
|
|
"completions/min_terminated_length": 152.7,
|
|
"entropy": 1.0965552292764187,
|
|
"epoch": 0.2611014066378597,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.10269385576248169,
|
|
"kl": 0.050288640061626214,
|
|
"learning_rate": 4.981319550016777e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 12324596.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.28629584312438966,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.2862958490848541,
|
|
"sampling/importance_sampling_ratio/max": 1.244599413871765,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998269200325012,
|
|
"sampling/importance_sampling_ratio/min": 0.7803301751613617,
|
|
"sampling/sampling_logp_difference/max": 0.2602036714553833,
|
|
"sampling/sampling_logp_difference/mean": 0.021378837898373602,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 333.1,
|
|
"completions/max_terminated_length": 333.1,
|
|
"completions/mean_length": 221.8625,
|
|
"completions/mean_terminated_length": 221.8625,
|
|
"completions/min_length": 135.2,
|
|
"completions/min_terminated_length": 135.2,
|
|
"entropy": 1.1194487735629082,
|
|
"epoch": 0.262020777787993,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.97575044631958,
|
|
"kl": 0.05989346381975338,
|
|
"learning_rate": 4.9810091266627835e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 12369417.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.3257186770439148,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.32571869492530825,
|
|
"sampling/importance_sampling_ratio/max": 1.2479507446289062,
|
|
"sampling/importance_sampling_ratio/mean": 0.999977195262909,
|
|
"sampling/importance_sampling_ratio/min": 0.75471031665802,
|
|
"sampling/sampling_logp_difference/max": 0.2949321448802948,
|
|
"sampling/sampling_logp_difference/mean": 0.021055320836603643,
|
|
"step": 2850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.5,
|
|
"completions/max_terminated_length": 278.5,
|
|
"completions/mean_length": 198.2375,
|
|
"completions/mean_terminated_length": 198.2375,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 1.0531558103859424,
|
|
"epoch": 0.26294014893812634,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.708948135375977,
|
|
"kl": 0.036559860524721446,
|
|
"learning_rate": 4.980696155113539e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 12412564.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3446744382381439,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.3446744501590729,
|
|
"sampling/importance_sampling_ratio/max": 1.2376705050468444,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998081743717193,
|
|
"sampling/importance_sampling_ratio/min": 0.7684375524520874,
|
|
"sampling/sampling_logp_difference/max": 0.27625452280044555,
|
|
"sampling/sampling_logp_difference/mean": 0.02165361903607845,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 259.4,
|
|
"completions/max_terminated_length": 259.4,
|
|
"completions/mean_length": 197.525,
|
|
"completions/mean_terminated_length": 197.525,
|
|
"completions/min_length": 137.5,
|
|
"completions/min_terminated_length": 137.5,
|
|
"entropy": 1.1611466467380525,
|
|
"epoch": 0.26385952008825964,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.18947243690490723,
|
|
"kl": 0.03421938415849581,
|
|
"learning_rate": 4.980380635690492e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 12456510.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.35464063584804534,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.3546406477689743,
|
|
"sampling/importance_sampling_ratio/max": 1.3299927473068238,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002754747867584,
|
|
"sampling/importance_sampling_ratio/min": 0.7863232970237732,
|
|
"sampling/sampling_logp_difference/max": 0.30860941410064696,
|
|
"sampling/sampling_logp_difference/mean": 0.022235395200550557,
|
|
"step": 2870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 264.2,
|
|
"completions/max_terminated_length": 264.2,
|
|
"completions/mean_length": 184.4875,
|
|
"completions/mean_terminated_length": 184.4875,
|
|
"completions/min_length": 122.0,
|
|
"completions/min_terminated_length": 122.0,
|
|
"entropy": 1.1276733413338662,
|
|
"epoch": 0.26477889123839293,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.813807010650635,
|
|
"kl": 0.055635694501688704,
|
|
"learning_rate": 4.980062568717704e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 12495869.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.3435225158929825,
|
|
"rewards/reward_fn/mean": 0.2125,
|
|
"rewards/reward_fn/std": 0.34352252185344695,
|
|
"sampling/importance_sampling_ratio/max": 1.2383883357048036,
|
|
"sampling/importance_sampling_ratio/mean": 0.9991217076778411,
|
|
"sampling/importance_sampling_ratio/min": 0.7905645251274109,
|
|
"sampling/sampling_logp_difference/max": 0.2475505828857422,
|
|
"sampling/sampling_logp_difference/mean": 0.0216397263109684,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 291.6,
|
|
"completions/max_terminated_length": 282.5,
|
|
"completions/mean_length": 204.55,
|
|
"completions/mean_terminated_length": 201.44642944335936,
|
|
"completions/min_length": 132.3,
|
|
"completions/min_terminated_length": 132.3,
|
|
"entropy": 1.0076976522803307,
|
|
"epoch": 0.26569826238852623,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1592673659324646,
|
|
"kl": 0.04144300357438624,
|
|
"learning_rate": 4.979741954521857e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 12537721.0,
|
|
"reward": 0.26875,
|
|
"reward_std": 0.355224609375,
|
|
"rewards/reward_fn/mean": 0.26875,
|
|
"rewards/reward_fn/std": 0.35522462129592897,
|
|
"sampling/importance_sampling_ratio/max": 1.2800498366355897,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999342679977417,
|
|
"sampling/importance_sampling_ratio/min": 0.7533438205718994,
|
|
"sampling/sampling_logp_difference/max": 0.3110794723033905,
|
|
"sampling/sampling_logp_difference/mean": 0.021210421435534955,
|
|
"step": 2890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.1,
|
|
"completions/max_terminated_length": 279.1,
|
|
"completions/mean_length": 184.8875,
|
|
"completions/mean_terminated_length": 184.8875,
|
|
"completions/min_length": 126.8,
|
|
"completions/min_terminated_length": 126.8,
|
|
"entropy": 0.9692872066050768,
|
|
"epoch": 0.2666176335386596,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.114948272705078,
|
|
"kl": 0.05371689720777795,
|
|
"learning_rate": 4.979418793432248e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 12581248.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.5004733771085739,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.5004734069108963,
|
|
"sampling/importance_sampling_ratio/max": 1.2769236803054809,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001907169818878,
|
|
"sampling/importance_sampling_ratio/min": 0.7716196656227112,
|
|
"sampling/sampling_logp_difference/max": 0.2743303418159485,
|
|
"sampling/sampling_logp_difference/mean": 0.020811568945646286,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.4,
|
|
"completions/max_terminated_length": 296.4,
|
|
"completions/mean_length": 213.3,
|
|
"completions/mean_terminated_length": 213.3,
|
|
"completions/min_length": 143.1,
|
|
"completions/min_terminated_length": 143.1,
|
|
"entropy": 1.0639724999666214,
|
|
"epoch": 0.2675370046887929,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.853530406951904,
|
|
"kl": 0.05208175663137808,
|
|
"learning_rate": 4.979093085780788e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 12627784.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.20580926835536956,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.20580926835536956,
|
|
"sampling/importance_sampling_ratio/max": 1.2603452920913696,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005508840084076,
|
|
"sampling/importance_sampling_ratio/min": 0.7826572835445404,
|
|
"sampling/sampling_logp_difference/max": 0.2577659010887146,
|
|
"sampling/sampling_logp_difference/mean": 0.02154218889772892,
|
|
"step": 2910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.0,
|
|
"completions/max_terminated_length": 304.0,
|
|
"completions/mean_length": 214.8375,
|
|
"completions/mean_terminated_length": 214.8375,
|
|
"completions/min_length": 132.7,
|
|
"completions/min_terminated_length": 132.7,
|
|
"entropy": 1.049601886421442,
|
|
"epoch": 0.2684563758389262,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.14672520756721497,
|
|
"kl": 0.058840165205765516,
|
|
"learning_rate": 4.978764831902005e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 12672515.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.31564186215400697,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.31564187407493594,
|
|
"sampling/importance_sampling_ratio/max": 1.269123363494873,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999415457248688,
|
|
"sampling/importance_sampling_ratio/min": 0.7534095704555511,
|
|
"sampling/sampling_logp_difference/max": 0.2956400036811829,
|
|
"sampling/sampling_logp_difference/mean": 0.020669311843812467,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 245.9,
|
|
"completions/max_terminated_length": 245.9,
|
|
"completions/mean_length": 177.8125,
|
|
"completions/mean_terminated_length": 177.8125,
|
|
"completions/min_length": 117.4,
|
|
"completions/min_terminated_length": 117.4,
|
|
"entropy": 1.0855563215911388,
|
|
"epoch": 0.2693757469890595,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.849121570587158,
|
|
"kl": 0.0681488914298825,
|
|
"learning_rate": 4.978434032133042e-06,
|
|
"loss": 0.0035,
|
|
"num_tokens": 12709956.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.42207511365413664,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.4220751315355301,
|
|
"sampling/importance_sampling_ratio/max": 1.2360840797424317,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993714809417724,
|
|
"sampling/importance_sampling_ratio/min": 0.7885816752910614,
|
|
"sampling/sampling_logp_difference/max": 0.25503069162368774,
|
|
"sampling/sampling_logp_difference/mean": 0.021899352408945562,
|
|
"step": 2930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.8,
|
|
"completions/max_terminated_length": 294.8,
|
|
"completions/mean_length": 222.85,
|
|
"completions/mean_terminated_length": 222.85,
|
|
"completions/min_length": 156.4,
|
|
"completions/min_terminated_length": 156.4,
|
|
"entropy": 1.036701576411724,
|
|
"epoch": 0.27029511813919277,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.780606269836426,
|
|
"kl": 0.047939900774508715,
|
|
"learning_rate": 4.978100686813658e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 12753416.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.39183932840824126,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.3918393403291702,
|
|
"sampling/importance_sampling_ratio/max": 1.2341225862503051,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999453306198121,
|
|
"sampling/importance_sampling_ratio/min": 0.7725021332502365,
|
|
"sampling/sampling_logp_difference/max": 0.281782329082489,
|
|
"sampling/sampling_logp_difference/mean": 0.020637968368828297,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.9,
|
|
"completions/max_terminated_length": 310.9,
|
|
"completions/mean_length": 213.575,
|
|
"completions/mean_terminated_length": 213.575,
|
|
"completions/min_length": 141.6,
|
|
"completions/min_terminated_length": 141.6,
|
|
"entropy": 1.2040755182504654,
|
|
"epoch": 0.2712144892893261,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.997399806976318,
|
|
"kl": 0.06263853986747563,
|
|
"learning_rate": 4.977764796286224e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 12798078.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.2205115258693695,
|
|
"sampling/importance_sampling_ratio/max": 1.2764033079147339,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998996794223786,
|
|
"sampling/importance_sampling_ratio/min": 0.7966888904571533,
|
|
"sampling/sampling_logp_difference/max": 0.2518589496612549,
|
|
"sampling/sampling_logp_difference/mean": 0.022214872390031816,
|
|
"step": 2950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.7,
|
|
"completions/max_terminated_length": 279.7,
|
|
"completions/mean_length": 201.2875,
|
|
"completions/mean_terminated_length": 201.2875,
|
|
"completions/min_length": 143.5,
|
|
"completions/min_terminated_length": 143.5,
|
|
"entropy": 1.107544044405222,
|
|
"epoch": 0.2721338604394594,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.1241946667432785,
|
|
"kl": 0.057853935711318624,
|
|
"learning_rate": 4.977426360895727e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 12840629.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.2612558871507645,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.26125589907169344,
|
|
"sampling/importance_sampling_ratio/max": 1.2495791912078857,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001029014587401,
|
|
"sampling/importance_sampling_ratio/min": 0.7849660515785217,
|
|
"sampling/sampling_logp_difference/max": 0.2641411066055298,
|
|
"sampling/sampling_logp_difference/mean": 0.0222238564863801,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 324.4,
|
|
"completions/max_terminated_length": 305.8,
|
|
"completions/mean_length": 215.55,
|
|
"completions/mean_terminated_length": 211.94464416503905,
|
|
"completions/min_length": 130.3,
|
|
"completions/min_terminated_length": 130.3,
|
|
"entropy": 0.9620839280076325,
|
|
"epoch": 0.2730532315895927,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.880609512329102,
|
|
"kl": 0.038240350448177196,
|
|
"learning_rate": 4.977085380989768e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 12883513.0,
|
|
"reward": 0.26875,
|
|
"reward_std": 0.3974449634552002,
|
|
"rewards/reward_fn/mean": 0.26875,
|
|
"rewards/reward_fn/std": 0.39744496941566465,
|
|
"sampling/importance_sampling_ratio/max": 1.2809149622917175,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003820061683655,
|
|
"sampling/importance_sampling_ratio/min": 0.7656985700130463,
|
|
"sampling/sampling_logp_difference/max": 0.3023351788520813,
|
|
"sampling/sampling_logp_difference/mean": 0.019760796707123517,
|
|
"step": 2970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.9,
|
|
"completions/max_terminated_length": 279.9,
|
|
"completions/mean_length": 207.2875,
|
|
"completions/mean_terminated_length": 207.2875,
|
|
"completions/min_length": 137.4,
|
|
"completions/min_terminated_length": 137.4,
|
|
"entropy": 1.024321373552084,
|
|
"epoch": 0.273972602739726,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.848912715911865,
|
|
"kl": 0.045333441102411595,
|
|
"learning_rate": 4.9767418569185585e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 12924784.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.4164423167705536,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.41644232273101806,
|
|
"sampling/importance_sampling_ratio/max": 1.2930202960968018,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998917162418366,
|
|
"sampling/importance_sampling_ratio/min": 0.7350365877151489,
|
|
"sampling/sampling_logp_difference/max": 0.3134430170059204,
|
|
"sampling/sampling_logp_difference/mean": 0.0207748519256711,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 332.4,
|
|
"completions/max_terminated_length": 332.4,
|
|
"completions/mean_length": 235.3,
|
|
"completions/mean_terminated_length": 235.3,
|
|
"completions/min_length": 149.1,
|
|
"completions/min_terminated_length": 149.1,
|
|
"entropy": 1.1033256374299527,
|
|
"epoch": 0.2748919738898593,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.912093639373779,
|
|
"kl": 0.04564914793591015,
|
|
"learning_rate": 4.9763957890349284e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 12971480.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.33515144884586334,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.33515146672725676,
|
|
"sampling/importance_sampling_ratio/max": 1.2637401580810548,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002837836742402,
|
|
"sampling/importance_sampling_ratio/min": 0.7673021078109741,
|
|
"sampling/sampling_logp_difference/max": 0.2827598750591278,
|
|
"sampling/sampling_logp_difference/mean": 0.02086910270154476,
|
|
"step": 2990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 338.4,
|
|
"completions/max_terminated_length": 313.9,
|
|
"completions/mean_length": 213.5375,
|
|
"completions/mean_terminated_length": 208.3446472167969,
|
|
"completions/min_length": 122.1,
|
|
"completions/min_terminated_length": 122.1,
|
|
"entropy": 1.1471606727689505,
|
|
"epoch": 0.27581134503999266,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.2565531730651855,
|
|
"kl": 0.030792669008951633,
|
|
"learning_rate": 4.976047177694315e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 13017027.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.37291836440563203,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.3729183703660965,
|
|
"sampling/importance_sampling_ratio/max": 1.255772602558136,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997790336608887,
|
|
"sampling/importance_sampling_ratio/min": 0.7934587657451629,
|
|
"sampling/sampling_logp_difference/max": 0.26108033657073976,
|
|
"sampling/sampling_logp_difference/mean": 0.021706547029316424,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.27581134503999266,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002181500872600349,
|
|
"eval_completions/max_length": 295.0732984293194,
|
|
"eval_completions/max_terminated_length": 293.2216404886562,
|
|
"eval_completions/mean_length": 202.85929319371726,
|
|
"eval_completions/mean_terminated_length": 202.31337797953822,
|
|
"eval_completions/min_length": 126.57766143106457,
|
|
"eval_completions/min_terminated_length": 126.57766143106457,
|
|
"eval_entropy": 1.0980144182632938,
|
|
"eval_frac_reward_zero_std": 0.24083769633507854,
|
|
"eval_kl": 0.03658795784053638,
|
|
"eval_loss": 0.0018208450637757778,
|
|
"eval_num_tokens": 13017027.0,
|
|
"eval_reward": 0.037303664921465966,
|
|
"eval_reward_std": 0.4122541890094418,
|
|
"eval_rewards/reward_fn/mean": 0.037303664921465966,
|
|
"eval_rewards/reward_fn/std": 0.41225420149208986,
|
|
"eval_runtime": 1721.6143,
|
|
"eval_samples_per_second": 0.333,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2836325915488065,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000328719095826,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7668360717737653,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2988978398615985,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021316910227581885,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.4,
|
|
"completions/max_terminated_length": 270.4,
|
|
"completions/mean_length": 192.3875,
|
|
"completions/mean_terminated_length": 192.3875,
|
|
"completions/min_length": 114.7,
|
|
"completions/min_terminated_length": 114.7,
|
|
"entropy": 1.0674329996109009,
|
|
"epoch": 0.27673071619012596,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.307253837585449,
|
|
"kl": 0.039002185128629206,
|
|
"learning_rate": 4.975696023254771e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 13055258.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.5295886754989624,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.5295886993408203,
|
|
"sampling/importance_sampling_ratio/max": 1.3257169485092164,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996242046356201,
|
|
"sampling/importance_sampling_ratio/min": 0.7877074241638183,
|
|
"sampling/sampling_logp_difference/max": 0.31526724100112913,
|
|
"sampling/sampling_logp_difference/mean": 0.021550440788269044,
|
|
"step": 3010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 342.9,
|
|
"completions/max_terminated_length": 310.7,
|
|
"completions/mean_length": 215.6,
|
|
"completions/mean_terminated_length": 209.03036041259764,
|
|
"completions/min_length": 124.2,
|
|
"completions/min_terminated_length": 124.2,
|
|
"entropy": 1.0915420100092887,
|
|
"epoch": 0.27765008734025926,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.749044895172119,
|
|
"kl": 0.05413867571041919,
|
|
"learning_rate": 4.97534232607696e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 13098818.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.5031409710645676,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.5031409829854965,
|
|
"sampling/importance_sampling_ratio/max": 1.2760480880737304,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998669803142548,
|
|
"sampling/importance_sampling_ratio/min": 0.7631634891033172,
|
|
"sampling/sampling_logp_difference/max": 0.29209101796150205,
|
|
"sampling/sampling_logp_difference/mean": 0.021610433980822564,
|
|
"step": 3020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.0,
|
|
"completions/max_terminated_length": 278.0,
|
|
"completions/mean_length": 195.575,
|
|
"completions/mean_terminated_length": 195.575,
|
|
"completions/min_length": 124.6,
|
|
"completions/min_terminated_length": 124.6,
|
|
"entropy": 1.048918904364109,
|
|
"epoch": 0.27856945849039255,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.53798246383667,
|
|
"kl": 0.0417088690912351,
|
|
"learning_rate": 4.974986086524156e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 13139744.0,
|
|
"reward": -0.14375,
|
|
"reward_std": 0.46859510242938995,
|
|
"rewards/reward_fn/mean": -0.14375,
|
|
"rewards/reward_fn/std": 0.4685951083898544,
|
|
"sampling/importance_sampling_ratio/max": 1.2597753643989562,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999534487724304,
|
|
"sampling/importance_sampling_ratio/min": 0.7457925081253052,
|
|
"sampling/sampling_logp_difference/max": 0.31349319219589233,
|
|
"sampling/sampling_logp_difference/mean": 0.020610079169273376,
|
|
"step": 3030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.0,
|
|
"completions/max_terminated_length": 287.0,
|
|
"completions/mean_length": 199.525,
|
|
"completions/mean_terminated_length": 199.525,
|
|
"completions/min_length": 117.5,
|
|
"completions/min_terminated_length": 117.5,
|
|
"entropy": 1.0540482595562934,
|
|
"epoch": 0.2794888296405259,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.633767604827881,
|
|
"kl": 0.033208812162047255,
|
|
"learning_rate": 4.974627304962247e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 13182074.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.5549404412508011,
|
|
"rewards/reward_fn/mean": 0.34375,
|
|
"rewards/reward_fn/std": 0.5549404472112656,
|
|
"sampling/importance_sampling_ratio/max": 1.2974908113479615,
|
|
"sampling/importance_sampling_ratio/mean": 0.999974274635315,
|
|
"sampling/importance_sampling_ratio/min": 0.8051841020584106,
|
|
"sampling/sampling_logp_difference/max": 0.2678371489048004,
|
|
"sampling/sampling_logp_difference/mean": 0.020697659999132156,
|
|
"step": 3040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 264.4,
|
|
"completions/max_terminated_length": 264.4,
|
|
"completions/mean_length": 179.4375,
|
|
"completions/mean_terminated_length": 179.4375,
|
|
"completions/min_length": 98.9,
|
|
"completions/min_terminated_length": 98.9,
|
|
"entropy": 1.0010355595499276,
|
|
"epoch": 0.2804082007906592,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.19045215845108032,
|
|
"kl": 0.028951244370546192,
|
|
"learning_rate": 4.97426598175973e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 13222797.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.3980518102645874,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.39805181622505187,
|
|
"sampling/importance_sampling_ratio/max": 1.2700633764266969,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003682076931,
|
|
"sampling/importance_sampling_ratio/min": 0.7980851233005524,
|
|
"sampling/sampling_logp_difference/max": 0.2582335829734802,
|
|
"sampling/sampling_logp_difference/mean": 0.019789416529238223,
|
|
"step": 3050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.8,
|
|
"completions/max_terminated_length": 285.8,
|
|
"completions/mean_length": 199.2375,
|
|
"completions/mean_terminated_length": 199.2375,
|
|
"completions/min_length": 111.8,
|
|
"completions/min_terminated_length": 111.8,
|
|
"entropy": 1.1236065208911896,
|
|
"epoch": 0.2813275719407925,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.808686256408691,
|
|
"kl": 0.04446217557415366,
|
|
"learning_rate": 4.973902117287714e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 13267168.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.48544313609600065,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.48544314205646516,
|
|
"sampling/importance_sampling_ratio/max": 1.2551715612411498,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999238669872283,
|
|
"sampling/importance_sampling_ratio/min": 0.7100574348121882,
|
|
"sampling/sampling_logp_difference/max": 0.5332852244377136,
|
|
"sampling/sampling_logp_difference/mean": 0.022360038198530673,
|
|
"step": 3060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.5,
|
|
"completions/max_terminated_length": 288.5,
|
|
"completions/mean_length": 190.3375,
|
|
"completions/mean_terminated_length": 190.3375,
|
|
"completions/min_length": 126.1,
|
|
"completions/min_terminated_length": 126.1,
|
|
"entropy": 1.0981203615665436,
|
|
"epoch": 0.2822469430909258,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11504124104976654,
|
|
"kl": 0.02490336262853816,
|
|
"learning_rate": 4.9735357119199154e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 13310067.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.2711646556854248,
|
|
"rewards/reward_fn/mean": 0.2125,
|
|
"rewards/reward_fn/std": 0.2711646556854248,
|
|
"sampling/importance_sampling_ratio/max": 1.2721507787704467,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998740136623383,
|
|
"sampling/importance_sampling_ratio/min": 0.8018085181713104,
|
|
"sampling/sampling_logp_difference/max": 0.25268272757530214,
|
|
"sampling/sampling_logp_difference/mean": 0.021480437740683554,
|
|
"step": 3070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.1,
|
|
"completions/max_terminated_length": 285.1,
|
|
"completions/mean_length": 204.1875,
|
|
"completions/mean_terminated_length": 204.1875,
|
|
"completions/min_length": 138.9,
|
|
"completions/min_terminated_length": 138.9,
|
|
"entropy": 1.120399809628725,
|
|
"epoch": 0.2831663142410591,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.795775413513184,
|
|
"kl": 0.029315559909446166,
|
|
"learning_rate": 4.9731667660326625e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 13351354.0,
|
|
"reward": -0.2,
|
|
"reward_std": 0.45687578022480013,
|
|
"rewards/reward_fn/mean": -0.2,
|
|
"rewards/reward_fn/std": 0.456875804066658,
|
|
"sampling/importance_sampling_ratio/max": 1.2691314101219178,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995928943157196,
|
|
"sampling/importance_sampling_ratio/min": 0.7629804670810699,
|
|
"sampling/sampling_logp_difference/max": 0.2955487906932831,
|
|
"sampling/sampling_logp_difference/mean": 0.021765482425689698,
|
|
"step": 3080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.7,
|
|
"completions/max_terminated_length": 282.7,
|
|
"completions/mean_length": 210.95,
|
|
"completions/mean_terminated_length": 210.95,
|
|
"completions/min_length": 142.4,
|
|
"completions/min_terminated_length": 142.4,
|
|
"entropy": 1.0443960294127463,
|
|
"epoch": 0.28408568539119244,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.066106796264648,
|
|
"kl": 0.026497692969860508,
|
|
"learning_rate": 4.972795280004893e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 13395502.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.33574098944664,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.33574100136756896,
|
|
"sampling/importance_sampling_ratio/max": 1.2967255115509033,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002476513385772,
|
|
"sampling/importance_sampling_ratio/min": 0.7796294748783111,
|
|
"sampling/sampling_logp_difference/max": 0.29486062526702883,
|
|
"sampling/sampling_logp_difference/mean": 0.021191072650253773,
|
|
"step": 3090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.0,
|
|
"completions/max_terminated_length": 291.0,
|
|
"completions/mean_length": 210.4625,
|
|
"completions/mean_terminated_length": 210.4625,
|
|
"completions/min_length": 140.0,
|
|
"completions/min_terminated_length": 140.0,
|
|
"entropy": 1.1657846421003342,
|
|
"epoch": 0.28500505654132574,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.7392778396606445,
|
|
"kl": 0.02931773483287543,
|
|
"learning_rate": 4.972421254218151e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 13438315.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.5231043308973312,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.5231043308973312,
|
|
"sampling/importance_sampling_ratio/max": 1.228700840473175,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998594582080841,
|
|
"sampling/importance_sampling_ratio/min": 0.787255734205246,
|
|
"sampling/sampling_logp_difference/max": 0.2500169575214386,
|
|
"sampling/sampling_logp_difference/mean": 0.02262396290898323,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 302.4,
|
|
"completions/max_terminated_length": 281.4,
|
|
"completions/mean_length": 202.0125,
|
|
"completions/mean_terminated_length": 198.0250015258789,
|
|
"completions/min_length": 133.3,
|
|
"completions/min_terminated_length": 133.3,
|
|
"entropy": 1.0418601922690869,
|
|
"epoch": 0.28592442769145904,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.446115016937256,
|
|
"kl": 0.042371876200195405,
|
|
"learning_rate": 4.972044689056593e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 13482820.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.2689625948667526,
|
|
"rewards/reward_fn/mean": 0.28125,
|
|
"rewards/reward_fn/std": 0.2689626067876816,
|
|
"sampling/importance_sampling_ratio/max": 1.2813395977020263,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995700478553772,
|
|
"sampling/importance_sampling_ratio/min": 0.7875003457069397,
|
|
"sampling/sampling_logp_difference/max": 0.2666512727737427,
|
|
"sampling/sampling_logp_difference/mean": 0.020998050272464753,
|
|
"step": 3110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.8,
|
|
"completions/max_terminated_length": 305.8,
|
|
"completions/mean_length": 203.325,
|
|
"completions/mean_terminated_length": 203.325,
|
|
"completions/min_length": 123.7,
|
|
"completions/min_terminated_length": 123.7,
|
|
"entropy": 0.9960630960762501,
|
|
"epoch": 0.28684379884159233,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.162259101867676,
|
|
"kl": 0.047720687976107,
|
|
"learning_rate": 4.97166558490698e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 13524614.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.5707158267498016,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.5707158446311951,
|
|
"sampling/importance_sampling_ratio/max": 1.2818321585655212,
|
|
"sampling/importance_sampling_ratio/mean": 1.000101637840271,
|
|
"sampling/importance_sampling_ratio/min": 0.7687285840511322,
|
|
"sampling/sampling_logp_difference/max": 0.28715428709983826,
|
|
"sampling/sampling_logp_difference/mean": 0.02036844249814749,
|
|
"step": 3120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 246.8,
|
|
"completions/max_terminated_length": 246.8,
|
|
"completions/mean_length": 180.5,
|
|
"completions/mean_terminated_length": 180.5,
|
|
"completions/min_length": 120.0,
|
|
"completions/min_terminated_length": 120.0,
|
|
"entropy": 0.9635615319013595,
|
|
"epoch": 0.2877631699917257,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.49933385848999,
|
|
"kl": 0.04054884716751985,
|
|
"learning_rate": 4.971283942158684e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 13564638.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.2799107313156128,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.2799107372760773,
|
|
"sampling/importance_sampling_ratio/max": 1.2627691984176637,
|
|
"sampling/importance_sampling_ratio/mean": 0.999916386604309,
|
|
"sampling/importance_sampling_ratio/min": 0.778308093547821,
|
|
"sampling/sampling_logp_difference/max": 0.2602915823459625,
|
|
"sampling/sampling_logp_difference/mean": 0.020179171673953535,
|
|
"step": 3130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.7,
|
|
"completions/max_terminated_length": 271.7,
|
|
"completions/mean_length": 181.7125,
|
|
"completions/mean_terminated_length": 181.7125,
|
|
"completions/min_length": 115.4,
|
|
"completions/min_terminated_length": 115.4,
|
|
"entropy": 1.0361908085644245,
|
|
"epoch": 0.288682541141859,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.2952147126197815,
|
|
"kl": 0.038313745649065825,
|
|
"learning_rate": 4.9708997612036805e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 13608391.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.4446160405874252,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.4446160525083542,
|
|
"sampling/importance_sampling_ratio/max": 1.2630624651908875,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997836589813233,
|
|
"sampling/importance_sampling_ratio/min": 0.7413627564907074,
|
|
"sampling/sampling_logp_difference/max": 0.30697921514511106,
|
|
"sampling/sampling_logp_difference/mean": 0.021165824867784976,
|
|
"step": 3140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 248.1,
|
|
"completions/max_terminated_length": 248.1,
|
|
"completions/mean_length": 178.725,
|
|
"completions/mean_terminated_length": 178.725,
|
|
"completions/min_length": 125.2,
|
|
"completions/min_terminated_length": 125.2,
|
|
"entropy": 0.9629081703722477,
|
|
"epoch": 0.2896019122919923,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.16850629448890686,
|
|
"kl": 0.04100114912143908,
|
|
"learning_rate": 4.970513042436556e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 13648793.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.29114731550216677,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.2911473214626312,
|
|
"sampling/importance_sampling_ratio/max": 1.2726138830184937,
|
|
"sampling/importance_sampling_ratio/mean": 1.000109314918518,
|
|
"sampling/importance_sampling_ratio/min": 0.7598125219345093,
|
|
"sampling/sampling_logp_difference/max": 0.3038113474845886,
|
|
"sampling/sampling_logp_difference/mean": 0.020257325284183024,
|
|
"step": 3150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.2,
|
|
"completions/max_terminated_length": 294.2,
|
|
"completions/mean_length": 205.925,
|
|
"completions/mean_terminated_length": 205.925,
|
|
"completions/min_length": 124.6,
|
|
"completions/min_terminated_length": 124.6,
|
|
"entropy": 1.0496474407613277,
|
|
"epoch": 0.2905212834421256,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.621676445007324,
|
|
"kl": 0.04431773464893922,
|
|
"learning_rate": 4.970123786254502e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 13691659.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.4355591177940369,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4355591356754303,
|
|
"sampling/importance_sampling_ratio/max": 1.2872658133506776,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999986410140991,
|
|
"sampling/importance_sampling_ratio/min": 0.742449676990509,
|
|
"sampling/sampling_logp_difference/max": 0.3295546770095825,
|
|
"sampling/sampling_logp_difference/mean": 0.0217421879991889,
|
|
"step": 3160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 294.2,
|
|
"completions/max_terminated_length": 281.4,
|
|
"completions/mean_length": 199.75,
|
|
"completions/mean_terminated_length": 196.63035888671874,
|
|
"completions/min_length": 112.5,
|
|
"completions/min_terminated_length": 112.5,
|
|
"entropy": 1.0945949390530587,
|
|
"epoch": 0.2914406545922589,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.85806131362915,
|
|
"kl": 0.03886735549895093,
|
|
"learning_rate": 4.969731993057313e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 13738151.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.3863959848880768,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.38639599084854126,
|
|
"sampling/importance_sampling_ratio/max": 1.2561033248901368,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004417181015015,
|
|
"sampling/importance_sampling_ratio/min": 0.7989996373653412,
|
|
"sampling/sampling_logp_difference/max": 0.2525927424430847,
|
|
"sampling/sampling_logp_difference/mean": 0.0211037527769804,
|
|
"step": 3170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.8,
|
|
"completions/max_terminated_length": 277.8,
|
|
"completions/mean_length": 179.85,
|
|
"completions/mean_terminated_length": 179.85,
|
|
"completions/min_length": 105.0,
|
|
"completions/min_terminated_length": 105.0,
|
|
"entropy": 1.0940855354070664,
|
|
"epoch": 0.2923600257423922,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.25357934832572937,
|
|
"kl": 0.04477006859378889,
|
|
"learning_rate": 4.969337663247395e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 13781563.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.2609549701213837,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.2609549820423126,
|
|
"sampling/importance_sampling_ratio/max": 1.3523677229881286,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998609185218811,
|
|
"sampling/importance_sampling_ratio/min": 0.734242606163025,
|
|
"sampling/sampling_logp_difference/max": 0.3794539928436279,
|
|
"sampling/sampling_logp_difference/mean": 0.021928014792501928,
|
|
"step": 3180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.8,
|
|
"completions/max_terminated_length": 309.8,
|
|
"completions/mean_length": 211.325,
|
|
"completions/mean_terminated_length": 211.325,
|
|
"completions/min_length": 131.7,
|
|
"completions/min_terminated_length": 131.7,
|
|
"entropy": 1.0738450296223163,
|
|
"epoch": 0.2932793968925255,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.378332614898682,
|
|
"kl": 0.0345492938242387,
|
|
"learning_rate": 4.968940797229756e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 13825949.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.339043465256691,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.33904347717761996,
|
|
"sampling/importance_sampling_ratio/max": 1.266939675807953,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004842698574066,
|
|
"sampling/importance_sampling_ratio/min": 0.7825249791145324,
|
|
"sampling/sampling_logp_difference/max": 0.2704328656196594,
|
|
"sampling/sampling_logp_difference/mean": 0.020584259554743765,
|
|
"step": 3190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.8,
|
|
"completions/max_terminated_length": 295.8,
|
|
"completions/mean_length": 202.4125,
|
|
"completions/mean_terminated_length": 202.4125,
|
|
"completions/min_length": 126.1,
|
|
"completions/min_terminated_length": 126.1,
|
|
"entropy": 1.1267855055630207,
|
|
"epoch": 0.2941987680426588,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.992135524749756,
|
|
"kl": 0.04341563874622807,
|
|
"learning_rate": 4.9685413954120064e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 13871142.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.4157505214214325,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.41575053334236145,
|
|
"sampling/importance_sampling_ratio/max": 1.3103078126907348,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004344403743743,
|
|
"sampling/importance_sampling_ratio/min": 0.7893327891826629,
|
|
"sampling/sampling_logp_difference/max": 0.2822605848312378,
|
|
"sampling/sampling_logp_difference/mean": 0.022518909722566604,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"epoch": 0.2941987680426588,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0015270506108202443,
|
|
"eval_completions/max_length": 284.97731239092496,
|
|
"eval_completions/max_terminated_length": 283.5340314136126,
|
|
"eval_completions/mean_length": 195.61649214659687,
|
|
"eval_completions/mean_terminated_length": 195.230699069837,
|
|
"eval_completions/min_length": 119.19371727748691,
|
|
"eval_completions/min_terminated_length": 119.19371727748691,
|
|
"eval_entropy": 1.1038762665439859,
|
|
"eval_frac_reward_zero_std": 0.24083769633507854,
|
|
"eval_kl": 0.038884694642512485,
|
|
"eval_loss": 0.0020698360167443752,
|
|
"eval_num_tokens": 13871142.0,
|
|
"eval_reward": 0.03130453752181501,
|
|
"eval_reward_std": 0.41291597225487126,
|
|
"eval_rewards/reward_fn/mean": 0.03130453752181501,
|
|
"eval_rewards/reward_fn/std": 0.4129159840093649,
|
|
"eval_runtime": 1673.7544,
|
|
"eval_samples_per_second": 0.342,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.261679679311383,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000500616602872,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7770163390856971,
|
|
"eval_sampling/sampling_logp_difference/max": 0.277808698424078,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021343533936270347,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.4,
|
|
"completions/max_terminated_length": 271.4,
|
|
"completions/mean_length": 179.8875,
|
|
"completions/mean_terminated_length": 179.8875,
|
|
"completions/min_length": 103.5,
|
|
"completions/min_terminated_length": 103.5,
|
|
"entropy": 1.0777474775910378,
|
|
"epoch": 0.2951181391927921,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.2479634284973145,
|
|
"kl": 0.0417698580364231,
|
|
"learning_rate": 4.9681394582043676e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 13911357.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.4260198652744293,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.4260198771953583,
|
|
"sampling/importance_sampling_ratio/max": 1.2817686200141907,
|
|
"sampling/importance_sampling_ratio/mean": 1.000111722946167,
|
|
"sampling/importance_sampling_ratio/min": 0.771096158027649,
|
|
"sampling/sampling_logp_difference/max": 0.30051048994064333,
|
|
"sampling/sampling_logp_difference/mean": 0.021910884976387025,
|
|
"step": 3210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.7,
|
|
"completions/max_terminated_length": 284.7,
|
|
"completions/mean_length": 199.6,
|
|
"completions/mean_terminated_length": 199.6,
|
|
"completions/min_length": 117.7,
|
|
"completions/min_terminated_length": 117.7,
|
|
"entropy": 1.09399051964283,
|
|
"epoch": 0.2960375103429254,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.04278564453125,
|
|
"kl": 0.03642093924572691,
|
|
"learning_rate": 4.967734986019659e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 13954469.0,
|
|
"reward": 0.06875,
|
|
"reward_std": 0.42089247405529023,
|
|
"rewards/reward_fn/mean": 0.06875,
|
|
"rewards/reward_fn/std": 0.42089249193668365,
|
|
"sampling/importance_sampling_ratio/max": 1.2835207343101502,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002387642860413,
|
|
"sampling/importance_sampling_ratio/min": 0.7825363755226136,
|
|
"sampling/sampling_logp_difference/max": 0.2811251699924469,
|
|
"sampling/sampling_logp_difference/mean": 0.02142675668001175,
|
|
"step": 3220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 254.5,
|
|
"completions/max_terminated_length": 254.5,
|
|
"completions/mean_length": 181.3625,
|
|
"completions/mean_terminated_length": 181.3625,
|
|
"completions/min_length": 112.7,
|
|
"completions/min_terminated_length": 112.7,
|
|
"entropy": 1.1572017669677734,
|
|
"epoch": 0.29695688149305877,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.134357288479805,
|
|
"kl": 0.12948516857577488,
|
|
"learning_rate": 4.967327979273307e-06,
|
|
"loss": 0.0061,
|
|
"num_tokens": 13997762.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.32480532228946685,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.3248053342103958,
|
|
"sampling/importance_sampling_ratio/max": 1.249702513217926,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000651717185973,
|
|
"sampling/importance_sampling_ratio/min": 0.7590961575508117,
|
|
"sampling/sampling_logp_difference/max": 0.33977370262145995,
|
|
"sampling/sampling_logp_difference/mean": 0.022720706462860108,
|
|
"step": 3230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 250.9,
|
|
"completions/max_terminated_length": 250.9,
|
|
"completions/mean_length": 176.85,
|
|
"completions/mean_terminated_length": 176.85,
|
|
"completions/min_length": 107.8,
|
|
"completions/min_terminated_length": 107.8,
|
|
"entropy": 1.0830637857317924,
|
|
"epoch": 0.29787625264319206,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.255032539367676,
|
|
"kl": 0.045315441233105956,
|
|
"learning_rate": 4.966918438383338e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 14035646.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.26680251955986023,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.2668025314807892,
|
|
"sampling/importance_sampling_ratio/max": 1.2868318915367127,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002416014671325,
|
|
"sampling/importance_sampling_ratio/min": 0.8168776869773865,
|
|
"sampling/sampling_logp_difference/max": 0.25513908863067625,
|
|
"sampling/sampling_logp_difference/mean": 0.021351755037903784,
|
|
"step": 3240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.8,
|
|
"completions/max_terminated_length": 305.8,
|
|
"completions/mean_length": 194.025,
|
|
"completions/mean_terminated_length": 194.025,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.028859481215477,
|
|
"epoch": 0.29879562379332536,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.21858549118042,
|
|
"kl": 0.04447895947378129,
|
|
"learning_rate": 4.966506363770387e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 14075936.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.4489682465791702,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.4489682585000992,
|
|
"sampling/importance_sampling_ratio/max": 1.2909308552742005,
|
|
"sampling/importance_sampling_ratio/mean": 1.000076675415039,
|
|
"sampling/importance_sampling_ratio/min": 0.7841595768928528,
|
|
"sampling/sampling_logp_difference/max": 0.28194546699523926,
|
|
"sampling/sampling_logp_difference/mean": 0.02058293465524912,
|
|
"step": 3250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.6,
|
|
"completions/max_terminated_length": 293.6,
|
|
"completions/mean_length": 197.5125,
|
|
"completions/mean_terminated_length": 197.5125,
|
|
"completions/min_length": 116.4,
|
|
"completions/min_terminated_length": 116.4,
|
|
"entropy": 1.0998571321368218,
|
|
"epoch": 0.29971499494345866,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.343313694000244,
|
|
"kl": 0.04879973778151907,
|
|
"learning_rate": 4.966091755857685e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 14118889.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.4374816298484802,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.4374816358089447,
|
|
"sampling/importance_sampling_ratio/max": 1.28458514213562,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001663267612457,
|
|
"sampling/importance_sampling_ratio/min": 0.7976416409015655,
|
|
"sampling/sampling_logp_difference/max": 0.271599555015564,
|
|
"sampling/sampling_logp_difference/mean": 0.020944624766707422,
|
|
"step": 3260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 334.6,
|
|
"completions/max_terminated_length": 334.6,
|
|
"completions/mean_length": 228.75,
|
|
"completions/mean_terminated_length": 228.75,
|
|
"completions/min_length": 124.4,
|
|
"completions/min_terminated_length": 124.4,
|
|
"entropy": 1.0862649217247964,
|
|
"epoch": 0.300634366093592,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.297386646270752,
|
|
"kl": 0.0435910573345609,
|
|
"learning_rate": 4.965674615071069e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 14164997.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.4481093943119049,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.44810941219329836,
|
|
"sampling/importance_sampling_ratio/max": 1.3015381574630738,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994955420494079,
|
|
"sampling/importance_sampling_ratio/min": 0.7816940486431122,
|
|
"sampling/sampling_logp_difference/max": 0.29544293880462646,
|
|
"sampling/sampling_logp_difference/mean": 0.020667477510869503,
|
|
"step": 3270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 275.2,
|
|
"completions/max_terminated_length": 265.8,
|
|
"completions/mean_length": 194.2875,
|
|
"completions/mean_terminated_length": 191.51071472167968,
|
|
"completions/min_length": 122.0,
|
|
"completions/min_terminated_length": 122.0,
|
|
"entropy": 1.1601298116147518,
|
|
"epoch": 0.3015537372437253,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.687579154968262,
|
|
"kl": 0.04985051549156196,
|
|
"learning_rate": 4.965254941838974e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 14206012.0,
|
|
"reward": -0.04375,
|
|
"reward_std": 0.40193670988082886,
|
|
"rewards/reward_fn/mean": -0.04375,
|
|
"rewards/reward_fn/std": 0.4019367277622223,
|
|
"sampling/importance_sampling_ratio/max": 1.2199535489082336,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005960702896117,
|
|
"sampling/importance_sampling_ratio/min": 0.7992093741893769,
|
|
"sampling/sampling_logp_difference/max": 0.22871973514556884,
|
|
"sampling/sampling_logp_difference/mean": 0.02164380457252264,
|
|
"step": 3280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.3,
|
|
"completions/max_terminated_length": 324.3,
|
|
"completions/mean_length": 216.7625,
|
|
"completions/mean_terminated_length": 216.7625,
|
|
"completions/min_length": 145.2,
|
|
"completions/min_terminated_length": 145.2,
|
|
"entropy": 1.1029285594820977,
|
|
"epoch": 0.3024731083938586,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.16830675303936005,
|
|
"kl": 0.0405906010302715,
|
|
"learning_rate": 4.964832736592441e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 14254009.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.3782377243041992,
|
|
"rewards/reward_fn/mean": 0.3625,
|
|
"rewards/reward_fn/std": 0.3782377302646637,
|
|
"sampling/importance_sampling_ratio/max": 1.2514019846916198,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997725665569306,
|
|
"sampling/importance_sampling_ratio/min": 0.7823267221450806,
|
|
"sampling/sampling_logp_difference/max": 0.2652637720108032,
|
|
"sampling/sampling_logp_difference/mean": 0.0212613919749856,
|
|
"step": 3290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.6,
|
|
"completions/max_terminated_length": 294.6,
|
|
"completions/mean_length": 202.2875,
|
|
"completions/mean_terminated_length": 202.2875,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 1.1082360297441483,
|
|
"epoch": 0.3033924795439919,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.044140815734863,
|
|
"kl": 0.04713789201341569,
|
|
"learning_rate": 4.964407999765105e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 14292768.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3900520890951157,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.39005209505558014,
|
|
"sampling/importance_sampling_ratio/max": 1.3164944648742676,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996932804584503,
|
|
"sampling/importance_sampling_ratio/min": 0.7677010238170624,
|
|
"sampling/sampling_logp_difference/max": 0.28633167743682864,
|
|
"sampling/sampling_logp_difference/mean": 0.021988416276872157,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 301.9,
|
|
"completions/max_terminated_length": 300.0,
|
|
"completions/mean_length": 220.2,
|
|
"completions/mean_terminated_length": 218.13392944335936,
|
|
"completions/min_length": 141.9,
|
|
"completions/min_terminated_length": 141.9,
|
|
"entropy": 1.112907313555479,
|
|
"epoch": 0.3043118506941252,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.377566814422607,
|
|
"kl": 0.03129078163765371,
|
|
"learning_rate": 4.963980731793208e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 14335296.0,
|
|
"reward": 0.11875,
|
|
"reward_std": 0.3474827975034714,
|
|
"rewards/reward_fn/mean": 0.11875,
|
|
"rewards/reward_fn/std": 0.34748280346393584,
|
|
"sampling/importance_sampling_ratio/max": 1.313843059539795,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002396404743195,
|
|
"sampling/importance_sampling_ratio/min": 0.7749020040035248,
|
|
"sampling/sampling_logp_difference/max": 0.28961069583892823,
|
|
"sampling/sampling_logp_difference/mean": 0.02126773204654455,
|
|
"step": 3310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.9,
|
|
"completions/max_terminated_length": 282.9,
|
|
"completions/mean_length": 209.5,
|
|
"completions/mean_terminated_length": 209.5,
|
|
"completions/min_length": 134.8,
|
|
"completions/min_terminated_length": 134.8,
|
|
"entropy": 1.0537394411861896,
|
|
"epoch": 0.30523122184425855,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.839500904083252,
|
|
"kl": 0.05472843888564967,
|
|
"learning_rate": 4.963550933115587e-06,
|
|
"loss": 0.0038,
|
|
"num_tokens": 14380368.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.3745658665895462,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.37456587255001067,
|
|
"sampling/importance_sampling_ratio/max": 1.2789030194282531,
|
|
"sampling/importance_sampling_ratio/mean": 1.000424474477768,
|
|
"sampling/importance_sampling_ratio/min": 0.7834398627281189,
|
|
"sampling/sampling_logp_difference/max": 0.2687701523303986,
|
|
"sampling/sampling_logp_difference/mean": 0.02134343031793833,
|
|
"step": 3320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 313.7,
|
|
"completions/max_terminated_length": 309.6,
|
|
"completions/mean_length": 226.8875,
|
|
"completions/mean_terminated_length": 219.82250061035157,
|
|
"completions/min_length": 137.7,
|
|
"completions/min_terminated_length": 137.7,
|
|
"entropy": 1.1447204016149044,
|
|
"epoch": 0.30615059299439185,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 4.403074264526367,
|
|
"kl": 0.04042053988669068,
|
|
"learning_rate": 4.96311860417368e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 14425839.0,
|
|
"reward": 0.14375,
|
|
"reward_std": 0.24405362606048583,
|
|
"rewards/reward_fn/mean": 0.14375,
|
|
"rewards/reward_fn/std": 0.2440536379814148,
|
|
"sampling/importance_sampling_ratio/max": 1.2183462262153626,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000001311302185,
|
|
"sampling/importance_sampling_ratio/min": 0.7743454873561859,
|
|
"sampling/sampling_logp_difference/max": 0.25811060667037966,
|
|
"sampling/sampling_logp_difference/mean": 0.021984154544770717,
|
|
"step": 3330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 349.0,
|
|
"completions/max_terminated_length": 340.3,
|
|
"completions/mean_length": 245.4375,
|
|
"completions/mean_terminated_length": 241.0625,
|
|
"completions/min_length": 163.3,
|
|
"completions/min_terminated_length": 163.3,
|
|
"entropy": 1.0474849876016379,
|
|
"epoch": 0.30706996414452514,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.259395599365234,
|
|
"kl": 0.045495229022344576,
|
|
"learning_rate": 4.962683745411524e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 14471722.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.48375589549541476,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.4837559133768082,
|
|
"sampling/importance_sampling_ratio/max": 1.261279320716858,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002941191196442,
|
|
"sampling/importance_sampling_ratio/min": 0.7963183403015137,
|
|
"sampling/sampling_logp_difference/max": 0.2499399185180664,
|
|
"sampling/sampling_logp_difference/mean": 0.02021382739767432,
|
|
"step": 3340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 341.9,
|
|
"completions/max_terminated_length": 324.4,
|
|
"completions/mean_length": 231.725,
|
|
"completions/mean_terminated_length": 228.94285888671874,
|
|
"completions/min_length": 129.1,
|
|
"completions/min_terminated_length": 129.1,
|
|
"entropy": 1.1465473890304565,
|
|
"epoch": 0.30798933529465844,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.311001300811768,
|
|
"kl": 0.040943041816353797,
|
|
"learning_rate": 4.9622463572757525e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 14518420.0,
|
|
"reward": -0.24375,
|
|
"reward_std": 0.3198355257511139,
|
|
"rewards/reward_fn/mean": -0.24375,
|
|
"rewards/reward_fn/std": 0.31983553767204287,
|
|
"sampling/importance_sampling_ratio/max": 1.2758788466453552,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996203184127808,
|
|
"sampling/importance_sampling_ratio/min": 0.7949548125267029,
|
|
"sampling/sampling_logp_difference/max": 0.2642476558685303,
|
|
"sampling/sampling_logp_difference/mean": 0.02134274486452341,
|
|
"step": 3350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.9,
|
|
"completions/max_terminated_length": 280.9,
|
|
"completions/mean_length": 201.7125,
|
|
"completions/mean_terminated_length": 201.7125,
|
|
"completions/min_length": 124.9,
|
|
"completions/min_terminated_length": 124.9,
|
|
"entropy": 1.0588207058608532,
|
|
"epoch": 0.30890870644479174,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 8.081878662109375,
|
|
"kl": 0.040357350173871966,
|
|
"learning_rate": 4.961806440215599e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 14563069.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.33921052813529967,
|
|
"rewards/reward_fn/mean": 0.2125,
|
|
"rewards/reward_fn/std": 0.3392105340957642,
|
|
"sampling/importance_sampling_ratio/max": 1.2543137669563293,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002131581306457,
|
|
"sampling/importance_sampling_ratio/min": 0.8036286413669587,
|
|
"sampling/sampling_logp_difference/max": 0.248527455329895,
|
|
"sampling/sampling_logp_difference/mean": 0.02072956208139658,
|
|
"step": 3360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.4,
|
|
"completions/max_terminated_length": 293.4,
|
|
"completions/mean_length": 222.625,
|
|
"completions/mean_terminated_length": 222.625,
|
|
"completions/min_length": 150.7,
|
|
"completions/min_terminated_length": 150.7,
|
|
"entropy": 1.1128535896539689,
|
|
"epoch": 0.3098280775949251,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.671053886413574,
|
|
"kl": 0.04595191022381186,
|
|
"learning_rate": 4.9613639946828925e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 14607599.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.35445033609867094,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.3544503539800644,
|
|
"sampling/importance_sampling_ratio/max": 1.2700241565704347,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999836146831512,
|
|
"sampling/importance_sampling_ratio/min": 0.7773639738559723,
|
|
"sampling/sampling_logp_difference/max": 0.27117196321487425,
|
|
"sampling/sampling_logp_difference/mean": 0.021323941834270953,
|
|
"step": 3370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 260.1,
|
|
"completions/max_terminated_length": 260.1,
|
|
"completions/mean_length": 180.65,
|
|
"completions/mean_terminated_length": 180.65,
|
|
"completions/min_length": 108.2,
|
|
"completions/min_terminated_length": 108.2,
|
|
"entropy": 1.037803926691413,
|
|
"epoch": 0.3107474487450584,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.516391277313232,
|
|
"kl": 0.057165876278304495,
|
|
"learning_rate": 4.960919021132062e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 14646387.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.5316522598266602,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.5316522598266602,
|
|
"sampling/importance_sampling_ratio/max": 1.3149107336997985,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002249658107758,
|
|
"sampling/importance_sampling_ratio/min": 0.7886558175086975,
|
|
"sampling/sampling_logp_difference/max": 0.29990922212600707,
|
|
"sampling/sampling_logp_difference/mean": 0.020648540277034045,
|
|
"step": 3380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.2,
|
|
"completions/max_terminated_length": 282.2,
|
|
"completions/mean_length": 206.2125,
|
|
"completions/mean_terminated_length": 206.2125,
|
|
"completions/min_length": 142.6,
|
|
"completions/min_terminated_length": 142.6,
|
|
"entropy": 1.0441970452666283,
|
|
"epoch": 0.3116668198951917,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 8.926807403564453,
|
|
"kl": 0.04298793796915561,
|
|
"learning_rate": 4.96047152002013e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 14692068.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.40020377933979034,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.40020379722118377,
|
|
"sampling/importance_sampling_ratio/max": 1.2929712414741517,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000506401062013,
|
|
"sampling/importance_sampling_ratio/min": 0.7757519900798797,
|
|
"sampling/sampling_logp_difference/max": 0.2902611494064331,
|
|
"sampling/sampling_logp_difference/mean": 0.02051110677421093,
|
|
"step": 3390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.2,
|
|
"completions/max_terminated_length": 270.2,
|
|
"completions/mean_length": 188.9375,
|
|
"completions/mean_terminated_length": 188.9375,
|
|
"completions/min_length": 125.0,
|
|
"completions/min_terminated_length": 125.0,
|
|
"entropy": 1.0867524527013301,
|
|
"epoch": 0.312586191045325,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.35761022567749,
|
|
"kl": 0.07332004045601934,
|
|
"learning_rate": 4.960021491806716e-06,
|
|
"loss": 0.004,
|
|
"num_tokens": 14731671.0,
|
|
"reward": -0.2625,
|
|
"reward_std": 0.3093190997838974,
|
|
"rewards/reward_fn/mean": -0.2625,
|
|
"rewards/reward_fn/std": 0.30931911170482634,
|
|
"sampling/importance_sampling_ratio/max": 1.2320275068283082,
|
|
"sampling/importance_sampling_ratio/mean": 0.999734616279602,
|
|
"sampling/importance_sampling_ratio/min": 0.7832122325897217,
|
|
"sampling/sampling_logp_difference/max": 0.25267844200134276,
|
|
"sampling/sampling_logp_difference/mean": 0.02092149220407009,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"epoch": 0.312586191045325,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0002181500872600349,
|
|
"eval_completions/max_length": 261.0314136125655,
|
|
"eval_completions/max_terminated_length": 260.97731239092496,
|
|
"eval_completions/mean_length": 184.41884816753927,
|
|
"eval_completions/mean_terminated_length": 184.37250687897102,
|
|
"eval_completions/min_length": 121.42931937172774,
|
|
"eval_completions/min_terminated_length": 121.42931937172774,
|
|
"eval_entropy": 1.1124632294577455,
|
|
"eval_frac_reward_zero_std": 0.38219895287958117,
|
|
"eval_kl": 0.06015212383173263,
|
|
"eval_loss": 0.0031121568754315376,
|
|
"eval_num_tokens": 14731671.0,
|
|
"eval_reward": 0.051919720767888304,
|
|
"eval_reward_std": 0.3012696447380758,
|
|
"eval_rewards/reward_fn/mean": 0.051919720767888304,
|
|
"eval_rewards/reward_fn/std": 0.3012696541000619,
|
|
"eval_runtime": 1552.7607,
|
|
"eval_samples_per_second": 0.369,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2540524825583785,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999092207737202,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7823862497927214,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2652849298081057,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021112393709728543,
|
|
"eval_steps_per_second": 0.046,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.1,
|
|
"completions/max_terminated_length": 290.1,
|
|
"completions/mean_length": 195.4125,
|
|
"completions/mean_terminated_length": 195.4125,
|
|
"completions/min_length": 128.0,
|
|
"completions/min_terminated_length": 128.0,
|
|
"entropy": 1.2019731745123863,
|
|
"epoch": 0.31350556219545833,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.16295911371707916,
|
|
"kl": 0.0738321740878746,
|
|
"learning_rate": 4.959568936954036e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 14774072.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.3311904340982437,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.3311904460191727,
|
|
"sampling/importance_sampling_ratio/max": 1.2507876396179198,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995803654193878,
|
|
"sampling/importance_sampling_ratio/min": 0.7372380375862122,
|
|
"sampling/sampling_logp_difference/max": 0.32430399060249326,
|
|
"sampling/sampling_logp_difference/mean": 0.022767294198274612,
|
|
"step": 3410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.2,
|
|
"completions/max_terminated_length": 283.2,
|
|
"completions/mean_length": 203.0125,
|
|
"completions/mean_terminated_length": 203.0125,
|
|
"completions/min_length": 133.9,
|
|
"completions/min_terminated_length": 133.9,
|
|
"entropy": 1.130712130665779,
|
|
"epoch": 0.31442493334559163,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.17660915851593018,
|
|
"kl": 0.05297590153058991,
|
|
"learning_rate": 4.959113855926898e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 14817921.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.28659676015377045,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.2865967661142349,
|
|
"sampling/importance_sampling_ratio/max": 1.2884628772735596,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003902316093445,
|
|
"sampling/importance_sampling_ratio/min": 0.752024120092392,
|
|
"sampling/sampling_logp_difference/max": 0.3201058626174927,
|
|
"sampling/sampling_logp_difference/mean": 0.0222373865544796,
|
|
"step": 3420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.1,
|
|
"completions/max_terminated_length": 281.1,
|
|
"completions/mean_length": 196.7875,
|
|
"completions/mean_terminated_length": 196.7875,
|
|
"completions/min_length": 118.8,
|
|
"completions/min_terminated_length": 118.8,
|
|
"entropy": 0.9948059290647506,
|
|
"epoch": 0.3153443044957249,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.125006675720215,
|
|
"kl": 0.0645312755019404,
|
|
"learning_rate": 4.958656249192707e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 14858296.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.25586686432361605,
|
|
"sampling/importance_sampling_ratio/max": 1.2303800344467164,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998889267444611,
|
|
"sampling/importance_sampling_ratio/min": 0.7863169848918915,
|
|
"sampling/sampling_logp_difference/max": 0.24640003442764283,
|
|
"sampling/sampling_logp_difference/mean": 0.020010964013636112,
|
|
"step": 3430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 260.5,
|
|
"completions/max_terminated_length": 260.5,
|
|
"completions/mean_length": 189.7875,
|
|
"completions/mean_terminated_length": 189.7875,
|
|
"completions/min_length": 125.4,
|
|
"completions/min_terminated_length": 125.4,
|
|
"entropy": 1.0337785191833972,
|
|
"epoch": 0.3162636756458582,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.000353813171387,
|
|
"kl": 0.056080954388016836,
|
|
"learning_rate": 4.958196117221464e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 14899375.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.3408251374959946,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.34082514345645903,
|
|
"sampling/importance_sampling_ratio/max": 1.251307439804077,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999927282333374,
|
|
"sampling/importance_sampling_ratio/min": 0.7130638420581817,
|
|
"sampling/sampling_logp_difference/max": 0.3540029048919678,
|
|
"sampling/sampling_logp_difference/mean": 0.021173002570867537,
|
|
"step": 3440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 253.0,
|
|
"completions/max_terminated_length": 253.0,
|
|
"completions/mean_length": 187.1,
|
|
"completions/mean_terminated_length": 187.1,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 1.08740364164114,
|
|
"epoch": 0.3171830467959915,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.1681165099143982,
|
|
"kl": 0.04227160386508331,
|
|
"learning_rate": 4.95773346048576e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 14942111.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.19231741726398469,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.19231742918491362,
|
|
"sampling/importance_sampling_ratio/max": 1.2743193864822389,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997759819030761,
|
|
"sampling/importance_sampling_ratio/min": 0.79029461145401,
|
|
"sampling/sampling_logp_difference/max": 0.27064874172210696,
|
|
"sampling/sampling_logp_difference/mean": 0.02145589329302311,
|
|
"step": 3450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.2,
|
|
"completions/max_terminated_length": 283.2,
|
|
"completions/mean_length": 192.6625,
|
|
"completions/mean_terminated_length": 192.6625,
|
|
"completions/min_length": 133.0,
|
|
"completions/min_terminated_length": 133.0,
|
|
"entropy": 1.1556284435093402,
|
|
"epoch": 0.31810241794612487,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 15.46902084350586,
|
|
"kl": 0.0817931110446807,
|
|
"learning_rate": 4.95726827946078e-06,
|
|
"loss": 0.0043,
|
|
"num_tokens": 14984036.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.15782093107700348,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.15782093703746797,
|
|
"sampling/importance_sampling_ratio/max": 1.2285220623016357,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999253928661347,
|
|
"sampling/importance_sampling_ratio/min": 0.7690546929836273,
|
|
"sampling/sampling_logp_difference/max": 0.27185516357421874,
|
|
"sampling/sampling_logp_difference/mean": 0.02233248669654131,
|
|
"step": 3460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.0,
|
|
"completions/max_terminated_length": 278.0,
|
|
"completions/mean_length": 210.0125,
|
|
"completions/mean_terminated_length": 210.0125,
|
|
"completions/min_length": 148.3,
|
|
"completions/min_terminated_length": 148.3,
|
|
"entropy": 1.1114678218960763,
|
|
"epoch": 0.31902178909625817,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.17906618118286133,
|
|
"kl": 0.05498292844276875,
|
|
"learning_rate": 4.956800574624303e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 15026229.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.2748226135969162,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.2748226195573807,
|
|
"sampling/importance_sampling_ratio/max": 1.2572020888328552,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999335944652558,
|
|
"sampling/importance_sampling_ratio/min": 0.7917510688304901,
|
|
"sampling/sampling_logp_difference/max": 0.2547708719968796,
|
|
"sampling/sampling_logp_difference/mean": 0.02142923753708601,
|
|
"step": 3470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.1,
|
|
"completions/max_terminated_length": 302.1,
|
|
"completions/mean_length": 202.375,
|
|
"completions/mean_terminated_length": 202.375,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.0744490265846252,
|
|
"epoch": 0.31994116024639147,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.356229305267334,
|
|
"kl": 0.08345635380828753,
|
|
"learning_rate": 4.9563303464567e-06,
|
|
"loss": 0.0037,
|
|
"num_tokens": 15065843.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.514201533794403,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.5142015397548676,
|
|
"sampling/importance_sampling_ratio/max": 1.2476582050323486,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003630936145782,
|
|
"sampling/importance_sampling_ratio/min": 0.7805294215679168,
|
|
"sampling/sampling_logp_difference/max": 0.26747645139694215,
|
|
"sampling/sampling_logp_difference/mean": 0.021488935872912406,
|
|
"step": 3480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.8,
|
|
"completions/max_terminated_length": 313.8,
|
|
"completions/mean_length": 202.15,
|
|
"completions/mean_terminated_length": 202.15,
|
|
"completions/min_length": 125.0,
|
|
"completions/min_terminated_length": 125.0,
|
|
"entropy": 1.1452050752937795,
|
|
"epoch": 0.32086053139652476,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.393003463745117,
|
|
"kl": 0.04011841090978123,
|
|
"learning_rate": 4.955857595440931e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 15111639.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.38434176445007323,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.3843417763710022,
|
|
"sampling/importance_sampling_ratio/max": 1.2546111941337585,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998025953769684,
|
|
"sampling/importance_sampling_ratio/min": 0.7932824790477753,
|
|
"sampling/sampling_logp_difference/max": 0.2517578363418579,
|
|
"sampling/sampling_logp_difference/mean": 0.021969419531524183,
|
|
"step": 3490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.4,
|
|
"completions/max_terminated_length": 298.4,
|
|
"completions/mean_length": 220.9,
|
|
"completions/mean_terminated_length": 220.9,
|
|
"completions/min_length": 145.8,
|
|
"completions/min_terminated_length": 145.8,
|
|
"entropy": 1.1159426502883434,
|
|
"epoch": 0.32177990254665806,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.783603191375732,
|
|
"kl": 0.0334494337788783,
|
|
"learning_rate": 4.955382322062551e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 15157527.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.384642681479454,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.384642693400383,
|
|
"sampling/importance_sampling_ratio/max": 1.239232099056244,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003385365009307,
|
|
"sampling/importance_sampling_ratio/min": 0.7998551845550537,
|
|
"sampling/sampling_logp_difference/max": 0.2339189827442169,
|
|
"sampling/sampling_logp_difference/mean": 0.021455386839807033,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.0,
|
|
"completions/max_terminated_length": 270.0,
|
|
"completions/mean_length": 193.4375,
|
|
"completions/mean_terminated_length": 193.4375,
|
|
"completions/min_length": 127.7,
|
|
"completions/min_terminated_length": 127.7,
|
|
"entropy": 1.0763651072978973,
|
|
"epoch": 0.3226992736967914,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 8.56969165802002,
|
|
"kl": 0.044204343925230205,
|
|
"learning_rate": 4.954904526809704e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 15197930.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.4350122928619385,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.43501230478286745,
|
|
"sampling/importance_sampling_ratio/max": 1.3894447207450866,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999247670173645,
|
|
"sampling/importance_sampling_ratio/min": 0.7801200091838837,
|
|
"sampling/sampling_logp_difference/max": 0.3426011085510254,
|
|
"sampling/sampling_logp_difference/mean": 0.02111690603196621,
|
|
"step": 3510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 265.9,
|
|
"completions/max_terminated_length": 265.9,
|
|
"completions/mean_length": 188.6125,
|
|
"completions/mean_terminated_length": 188.6125,
|
|
"completions/min_length": 118.4,
|
|
"completions/min_terminated_length": 118.4,
|
|
"entropy": 1.0959118843078612,
|
|
"epoch": 0.3236186448469247,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.62729024887085,
|
|
"kl": 0.027812705538235605,
|
|
"learning_rate": 4.954424210173122e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 15239739.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.413811257481575,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.41381126940250396,
|
|
"sampling/importance_sampling_ratio/max": 1.319244658946991,
|
|
"sampling/importance_sampling_ratio/mean": 1.000618177652359,
|
|
"sampling/importance_sampling_ratio/min": 0.7903915464878082,
|
|
"sampling/sampling_logp_difference/max": 0.2810153245925903,
|
|
"sampling/sampling_logp_difference/mean": 0.021199222840368747,
|
|
"step": 3520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 295.9,
|
|
"completions/max_terminated_length": 283.7,
|
|
"completions/mean_length": 200.2125,
|
|
"completions/mean_terminated_length": 196.90714416503906,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.0972279332578183,
|
|
"epoch": 0.324538015997058,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.982865810394287,
|
|
"kl": 0.03530151855084114,
|
|
"learning_rate": 4.95394137264613e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 15280564.0,
|
|
"reward": 0.19375,
|
|
"reward_std": 0.4013991504907608,
|
|
"rewards/reward_fn/mean": 0.19375,
|
|
"rewards/reward_fn/std": 0.40139917433261874,
|
|
"sampling/importance_sampling_ratio/max": 1.2826046228408814,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995478928089142,
|
|
"sampling/importance_sampling_ratio/min": 0.7916634500026702,
|
|
"sampling/sampling_logp_difference/max": 0.2662949085235596,
|
|
"sampling/sampling_logp_difference/mean": 0.020691893808543683,
|
|
"step": 3530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 267.1,
|
|
"completions/max_terminated_length": 267.1,
|
|
"completions/mean_length": 183.35,
|
|
"completions/mean_terminated_length": 183.35,
|
|
"completions/min_length": 121.6,
|
|
"completions/min_terminated_length": 121.6,
|
|
"entropy": 1.1036290675401688,
|
|
"epoch": 0.3254573871471913,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.13474500179290771,
|
|
"kl": 0.03013908474240452,
|
|
"learning_rate": 4.953456014724643e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 15317968.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.3964372009038925,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.396437206864357,
|
|
"sampling/importance_sampling_ratio/max": 1.3327782154083252,
|
|
"sampling/importance_sampling_ratio/mean": 1.000079196691513,
|
|
"sampling/importance_sampling_ratio/min": 0.7919772505760193,
|
|
"sampling/sampling_logp_difference/max": 0.3088480710983276,
|
|
"sampling/sampling_logp_difference/mean": 0.021486950293183326,
|
|
"step": 3540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.8,
|
|
"completions/max_terminated_length": 272.8,
|
|
"completions/mean_length": 189.6,
|
|
"completions/mean_terminated_length": 189.6,
|
|
"completions/min_length": 125.0,
|
|
"completions/min_terminated_length": 125.0,
|
|
"entropy": 1.114840267598629,
|
|
"epoch": 0.32637675829732465,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.2900848984718323,
|
|
"kl": 0.025565692828968166,
|
|
"learning_rate": 4.952968136907158e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 15358128.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.3709822952747345,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.370982301235199,
|
|
"sampling/importance_sampling_ratio/max": 1.2497904300689697,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997704565525055,
|
|
"sampling/importance_sampling_ratio/min": 0.7842993438243866,
|
|
"sampling/sampling_logp_difference/max": 0.2657679557800293,
|
|
"sampling/sampling_logp_difference/mean": 0.021084331348538397,
|
|
"step": 3550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 319.0,
|
|
"completions/max_terminated_length": 311.3,
|
|
"completions/mean_length": 228.05,
|
|
"completions/mean_terminated_length": 225.3107147216797,
|
|
"completions/min_length": 144.7,
|
|
"completions/min_terminated_length": 144.7,
|
|
"entropy": 1.124162356555462,
|
|
"epoch": 0.32729612944745795,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1541113704442978,
|
|
"kl": 0.03783324921969324,
|
|
"learning_rate": 4.952477739694768e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 15405028.0,
|
|
"reward": -0.03125,
|
|
"reward_std": 0.3908640146255493,
|
|
"rewards/reward_fn/mean": -0.03125,
|
|
"rewards/reward_fn/std": 0.3908640265464783,
|
|
"sampling/importance_sampling_ratio/max": 1.2465819478034974,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000207543373107,
|
|
"sampling/importance_sampling_ratio/min": 0.796151626110077,
|
|
"sampling/sampling_logp_difference/max": 0.24124348163604736,
|
|
"sampling/sampling_logp_difference/mean": 0.021043410524725914,
|
|
"step": 3560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.8,
|
|
"completions/max_terminated_length": 279.8,
|
|
"completions/mean_length": 212.4125,
|
|
"completions/mean_terminated_length": 212.4125,
|
|
"completions/min_length": 145.3,
|
|
"completions/min_terminated_length": 145.3,
|
|
"entropy": 1.2658915683627128,
|
|
"epoch": 0.32821550059759125,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.551015377044678,
|
|
"kl": 0.0411328959162347,
|
|
"learning_rate": 4.951984823591148e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 15449293.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.4665554463863373,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.46655545234680174,
|
|
"sampling/importance_sampling_ratio/max": 1.2340397000312806,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001348793506621,
|
|
"sampling/importance_sampling_ratio/min": 0.7853639185428619,
|
|
"sampling/sampling_logp_difference/max": 0.25348862409591677,
|
|
"sampling/sampling_logp_difference/mean": 0.02258206345140934,
|
|
"step": 3570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 259.2,
|
|
"completions/max_terminated_length": 259.2,
|
|
"completions/mean_length": 191.6875,
|
|
"completions/mean_terminated_length": 191.6875,
|
|
"completions/min_length": 143.8,
|
|
"completions/min_terminated_length": 143.8,
|
|
"entropy": 1.1672445468604564,
|
|
"epoch": 0.32913487174772454,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.156494617462158,
|
|
"kl": 0.043730844254605473,
|
|
"learning_rate": 4.951489389102564e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 15490532.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.3647657185792923,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.36476574242115023,
|
|
"sampling/importance_sampling_ratio/max": 1.2308570504188538,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994194984436036,
|
|
"sampling/importance_sampling_ratio/min": 0.7789407908916474,
|
|
"sampling/sampling_logp_difference/max": 0.2654020071029663,
|
|
"sampling/sampling_logp_difference/mean": 0.022163640707731247,
|
|
"step": 3580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.1,
|
|
"completions/max_terminated_length": 301.1,
|
|
"completions/mean_length": 205.275,
|
|
"completions/mean_terminated_length": 205.275,
|
|
"completions/min_length": 130.7,
|
|
"completions/min_terminated_length": 130.7,
|
|
"entropy": 1.1414539583027363,
|
|
"epoch": 0.33005424289785784,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.749905109405518,
|
|
"kl": 0.039948835014365615,
|
|
"learning_rate": 4.950991436737868e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 15534834.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.43887105882167815,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.4388710707426071,
|
|
"sampling/importance_sampling_ratio/max": 1.2631184101104735,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997998237609863,
|
|
"sampling/importance_sampling_ratio/min": 0.7739748060703278,
|
|
"sampling/sampling_logp_difference/max": 0.29264225959777834,
|
|
"sampling/sampling_logp_difference/mean": 0.02140676397830248,
|
|
"step": 3590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.6,
|
|
"completions/max_terminated_length": 274.6,
|
|
"completions/mean_length": 195.7625,
|
|
"completions/mean_terminated_length": 195.7625,
|
|
"completions/min_length": 125.3,
|
|
"completions/min_terminated_length": 125.3,
|
|
"entropy": 1.1062183894217015,
|
|
"epoch": 0.3309736140479912,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.103593826293945,
|
|
"kl": 0.04652380687184632,
|
|
"learning_rate": 4.9504909670084934e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 15578039.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.3500634729862213,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.3500634849071503,
|
|
"sampling/importance_sampling_ratio/max": 1.2819628834724426,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000762343406677,
|
|
"sampling/importance_sampling_ratio/min": 0.7846461653709411,
|
|
"sampling/sampling_logp_difference/max": 0.2850764274597168,
|
|
"sampling/sampling_logp_difference/mean": 0.0208723871037364,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"epoch": 0.3309736140479912,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0028359511343804536,
|
|
"eval_completions/max_length": 294.4938917975567,
|
|
"eval_completions/max_terminated_length": 291.40488656195464,
|
|
"eval_completions/mean_length": 202.26657940663176,
|
|
"eval_completions/mean_terminated_length": 201.4967902751077,
|
|
"eval_completions/min_length": 127.19371727748691,
|
|
"eval_completions/min_terminated_length": 127.19371727748691,
|
|
"eval_entropy": 1.1608602334899636,
|
|
"eval_frac_reward_zero_std": 0.28272251308900526,
|
|
"eval_kl": 0.04071375128876476,
|
|
"eval_loss": 0.0020598764531314373,
|
|
"eval_num_tokens": 15578039.0,
|
|
"eval_reward": 0.04253926701570681,
|
|
"eval_reward_std": 0.3759350982636057,
|
|
"eval_rewards/reward_fn/mean": 0.04253926701570681,
|
|
"eval_rewards/reward_fn/std": 0.3759351092899449,
|
|
"eval_runtime": 1719.186,
|
|
"eval_samples_per_second": 0.333,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2685776919802654,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999945341604543,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7839648288529997,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2756668614779467,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02123409967355911,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 333.3,
|
|
"completions/max_terminated_length": 310.8,
|
|
"completions/mean_length": 217.575,
|
|
"completions/mean_terminated_length": 214.05535888671875,
|
|
"completions/min_length": 139.3,
|
|
"completions/min_terminated_length": 139.3,
|
|
"entropy": 1.180570238083601,
|
|
"epoch": 0.3318929851981245,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.21361000835895538,
|
|
"kl": 0.031616039772052316,
|
|
"learning_rate": 4.949987980428466e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 15626413.0,
|
|
"reward": 0.01875,
|
|
"reward_std": 0.33154732286930083,
|
|
"rewards/reward_fn/mean": 0.01875,
|
|
"rewards/reward_fn/std": 0.3315473347902298,
|
|
"sampling/importance_sampling_ratio/max": 1.2511038780212402,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998937308788299,
|
|
"sampling/importance_sampling_ratio/min": 0.8101923346519471,
|
|
"sampling/sampling_logp_difference/max": 0.2298378348350525,
|
|
"sampling/sampling_logp_difference/mean": 0.021659868955612182,
|
|
"step": 3610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.3,
|
|
"completions/max_terminated_length": 314.3,
|
|
"completions/mean_length": 217.55,
|
|
"completions/mean_terminated_length": 217.55,
|
|
"completions/min_length": 129.4,
|
|
"completions/min_terminated_length": 129.4,
|
|
"entropy": 1.1649263449013234,
|
|
"epoch": 0.3328123563482578,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.513465404510498,
|
|
"kl": 0.04388266638852656,
|
|
"learning_rate": 4.949482477514391e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 15670633.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.3458986967802048,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.34589870274066925,
|
|
"sampling/importance_sampling_ratio/max": 1.33402818441391,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993565559387207,
|
|
"sampling/importance_sampling_ratio/min": 0.7917412281036377,
|
|
"sampling/sampling_logp_difference/max": 0.3039770245552063,
|
|
"sampling/sampling_logp_difference/mean": 0.021887423284351826,
|
|
"step": 3620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 303.4,
|
|
"completions/max_terminated_length": 301.5,
|
|
"completions/mean_length": 215.1875,
|
|
"completions/mean_terminated_length": 213.78571472167968,
|
|
"completions/min_length": 129.0,
|
|
"completions/min_terminated_length": 129.0,
|
|
"entropy": 1.0215728491544724,
|
|
"epoch": 0.3337317274983911,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.613890647888184,
|
|
"kl": 0.040379841323010625,
|
|
"learning_rate": 4.948974458785463e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 15711720.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.4271946668624878,
|
|
"rewards/reward_fn/mean": 0.09375,
|
|
"rewards/reward_fn/std": 0.42719467878341677,
|
|
"sampling/importance_sampling_ratio/max": 1.297000002861023,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994298100471497,
|
|
"sampling/importance_sampling_ratio/min": 0.7874313831329346,
|
|
"sampling/sampling_logp_difference/max": 0.2804200530052185,
|
|
"sampling/sampling_logp_difference/mean": 0.020793511904776095,
|
|
"step": 3630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.9,
|
|
"completions/max_terminated_length": 268.9,
|
|
"completions/mean_length": 190.925,
|
|
"completions/mean_terminated_length": 190.925,
|
|
"completions/min_length": 120.7,
|
|
"completions/min_terminated_length": 120.7,
|
|
"entropy": 1.1066973432898521,
|
|
"epoch": 0.33465109864852444,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.15176279842853546,
|
|
"kl": 0.03049780459841713,
|
|
"learning_rate": 4.9484639247634555e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 15755402.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.2394672751426697,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.23946728110313414,
|
|
"sampling/importance_sampling_ratio/max": 1.2649291038513184,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996630251407623,
|
|
"sampling/importance_sampling_ratio/min": 0.7570954263210297,
|
|
"sampling/sampling_logp_difference/max": 0.3304596126079559,
|
|
"sampling/sampling_logp_difference/mean": 0.022077969275414944,
|
|
"step": 3640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.9,
|
|
"completions/max_terminated_length": 288.9,
|
|
"completions/mean_length": 210.2875,
|
|
"completions/mean_terminated_length": 210.2875,
|
|
"completions/min_length": 134.3,
|
|
"completions/min_terminated_length": 134.3,
|
|
"entropy": 1.1345528542995453,
|
|
"epoch": 0.33557046979865773,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.1285932958126068,
|
|
"kl": 0.044535727973561735,
|
|
"learning_rate": 4.947950875972729e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 15801385.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.2540946453809738,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.2540946573019028,
|
|
"sampling/importance_sampling_ratio/max": 1.2794812321662903,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000095248222352,
|
|
"sampling/importance_sampling_ratio/min": 0.7962130904197693,
|
|
"sampling/sampling_logp_difference/max": 0.265420126914978,
|
|
"sampling/sampling_logp_difference/mean": 0.021642114501446486,
|
|
"step": 3650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.6,
|
|
"completions/max_terminated_length": 305.6,
|
|
"completions/mean_length": 217.9625,
|
|
"completions/mean_terminated_length": 217.9625,
|
|
"completions/min_length": 147.8,
|
|
"completions/min_terminated_length": 147.8,
|
|
"entropy": 1.087868282943964,
|
|
"epoch": 0.33648984094879103,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.727581977844238,
|
|
"kl": 0.04330416562734172,
|
|
"learning_rate": 4.947435312940226e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 15845302.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.3432215988636017,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.34322160482406616,
|
|
"sampling/importance_sampling_ratio/max": 1.2704196572303772,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999796926975251,
|
|
"sampling/importance_sampling_ratio/min": 0.7525640308856965,
|
|
"sampling/sampling_logp_difference/max": 0.3059571862220764,
|
|
"sampling/sampling_logp_difference/mean": 0.021398631483316423,
|
|
"step": 3660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 256.1,
|
|
"completions/max_terminated_length": 256.1,
|
|
"completions/mean_length": 189.8875,
|
|
"completions/mean_terminated_length": 189.8875,
|
|
"completions/min_length": 126.6,
|
|
"completions/min_terminated_length": 126.6,
|
|
"entropy": 1.0530556544661522,
|
|
"epoch": 0.3374092120989243,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.61439323425293,
|
|
"kl": 0.02843841178691946,
|
|
"learning_rate": 4.94691723619547e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 15887277.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.267578649520874,
|
|
"rewards/reward_fn/mean": 0.475,
|
|
"rewards/reward_fn/std": 0.267578661441803,
|
|
"sampling/importance_sampling_ratio/max": 1.2762251019477844,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003567576408385,
|
|
"sampling/importance_sampling_ratio/min": 0.7683699250221252,
|
|
"sampling/sampling_logp_difference/max": 0.30725838541984557,
|
|
"sampling/sampling_logp_difference/mean": 0.020744536258280277,
|
|
"step": 3670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 299.2,
|
|
"completions/max_terminated_length": 279.2,
|
|
"completions/mean_length": 200.05,
|
|
"completions/mean_terminated_length": 196.35714416503907,
|
|
"completions/min_length": 117.8,
|
|
"completions/min_terminated_length": 117.8,
|
|
"entropy": 1.0470870293676853,
|
|
"epoch": 0.3383285832490576,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.2421112060546875,
|
|
"kl": 0.027060857601463796,
|
|
"learning_rate": 4.946396646270569e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 15934937.0,
|
|
"reward": 0.19375,
|
|
"reward_std": 0.4837629318237305,
|
|
"rewards/reward_fn/mean": 0.19375,
|
|
"rewards/reward_fn/std": 0.48376294374465945,
|
|
"sampling/importance_sampling_ratio/max": 1.2478211402893067,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000157594680785,
|
|
"sampling/importance_sampling_ratio/min": 0.7879285931587219,
|
|
"sampling/sampling_logp_difference/max": 0.24876997470855713,
|
|
"sampling/sampling_logp_difference/mean": 0.020888124220073225,
|
|
"step": 3680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.7,
|
|
"completions/max_terminated_length": 255.7,
|
|
"completions/mean_length": 180.6625,
|
|
"completions/mean_terminated_length": 180.6625,
|
|
"completions/min_length": 113.2,
|
|
"completions/min_terminated_length": 113.2,
|
|
"entropy": 1.0996396481990813,
|
|
"epoch": 0.339247954399191,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.609167575836182,
|
|
"kl": 0.027532691974192856,
|
|
"learning_rate": 4.945873543700209e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 15975270.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.49643046259880064,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.49643046855926515,
|
|
"sampling/importance_sampling_ratio/max": 1.2901219129562378,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997725486755371,
|
|
"sampling/importance_sampling_ratio/min": 0.7695867896080018,
|
|
"sampling/sampling_logp_difference/max": 0.3122891902923584,
|
|
"sampling/sampling_logp_difference/mean": 0.021773027069866658,
|
|
"step": 3690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.1,
|
|
"completions/max_terminated_length": 287.1,
|
|
"completions/mean_length": 189.5375,
|
|
"completions/mean_terminated_length": 189.5375,
|
|
"completions/min_length": 124.0,
|
|
"completions/min_terminated_length": 124.0,
|
|
"entropy": 1.1054870411753654,
|
|
"epoch": 0.3401673255493243,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.653794288635254,
|
|
"kl": 0.034996135684195906,
|
|
"learning_rate": 4.945347929021659e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 16018065.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.445802640914917,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.445802640914917,
|
|
"sampling/importance_sampling_ratio/max": 1.2407708287239074,
|
|
"sampling/importance_sampling_ratio/mean": 1.000261563062668,
|
|
"sampling/importance_sampling_ratio/min": 0.7980013370513916,
|
|
"sampling/sampling_logp_difference/max": 0.2325606346130371,
|
|
"sampling/sampling_logp_difference/mean": 0.021481046080589296,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 245.1,
|
|
"completions/max_terminated_length": 245.1,
|
|
"completions/mean_length": 182.3625,
|
|
"completions/mean_terminated_length": 182.3625,
|
|
"completions/min_length": 118.2,
|
|
"completions/min_terminated_length": 118.2,
|
|
"entropy": 1.0582818120718003,
|
|
"epoch": 0.34108669669945757,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.14568209648132324,
|
|
"kl": 0.027012444072170183,
|
|
"learning_rate": 4.944819802774767e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 16056830.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.1974882572889328,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.19748826324939728,
|
|
"sampling/importance_sampling_ratio/max": 1.2445526838302612,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001866579055787,
|
|
"sampling/importance_sampling_ratio/min": 0.7962485194206238,
|
|
"sampling/sampling_logp_difference/max": 0.24086624383926392,
|
|
"sampling/sampling_logp_difference/mean": 0.021006628498435022,
|
|
"step": 3710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.1,
|
|
"completions/max_terminated_length": 308.1,
|
|
"completions/mean_length": 203.5625,
|
|
"completions/mean_terminated_length": 203.5625,
|
|
"completions/min_length": 132.1,
|
|
"completions/min_terminated_length": 132.1,
|
|
"entropy": 1.1114559836685658,
|
|
"epoch": 0.34200606784959087,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 4.144114971160889,
|
|
"kl": 0.021790811145910992,
|
|
"learning_rate": 4.944289165501961e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 16099795.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.14511305689811707,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.14511306285858155,
|
|
"sampling/importance_sampling_ratio/max": 1.281035840511322,
|
|
"sampling/importance_sampling_ratio/mean": 1.000019919872284,
|
|
"sampling/importance_sampling_ratio/min": 0.7919380784034729,
|
|
"sampling/sampling_logp_difference/max": 0.27017725706100465,
|
|
"sampling/sampling_logp_difference/mean": 0.021005408093333244,
|
|
"step": 3720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.8,
|
|
"completions/max_terminated_length": 295.8,
|
|
"completions/mean_length": 206.225,
|
|
"completions/mean_terminated_length": 206.225,
|
|
"completions/min_length": 127.1,
|
|
"completions/min_terminated_length": 127.1,
|
|
"entropy": 1.0347593866288662,
|
|
"epoch": 0.34292543899972416,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.044461727142334,
|
|
"kl": 0.0385551703220699,
|
|
"learning_rate": 4.9437560177482494e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 16140285.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.47564709186553955,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4756471037864685,
|
|
"sampling/importance_sampling_ratio/max": 1.3385959029197694,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997911810874939,
|
|
"sampling/importance_sampling_ratio/min": 0.7981269598007202,
|
|
"sampling/sampling_logp_difference/max": 0.2989547193050385,
|
|
"sampling/sampling_logp_difference/mean": 0.020690604858100415,
|
|
"step": 3730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.6,
|
|
"completions/max_terminated_length": 306.6,
|
|
"completions/mean_length": 212.2,
|
|
"completions/mean_terminated_length": 212.2,
|
|
"completions/min_length": 130.9,
|
|
"completions/min_terminated_length": 130.9,
|
|
"entropy": 1.1073269113898276,
|
|
"epoch": 0.3438448101498575,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.492613315582275,
|
|
"kl": 0.03056586690363474,
|
|
"learning_rate": 4.943220360061215e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 16185741.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.49708993136882784,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.49708996117115023,
|
|
"sampling/importance_sampling_ratio/max": 1.2766258239746093,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999300539493561,
|
|
"sampling/importance_sampling_ratio/min": 0.7869200527667999,
|
|
"sampling/sampling_logp_difference/max": 0.26742135286331176,
|
|
"sampling/sampling_logp_difference/mean": 0.020869196392595768,
|
|
"step": 3740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.0,
|
|
"completions/max_terminated_length": 302.0,
|
|
"completions/mean_length": 212.825,
|
|
"completions/mean_terminated_length": 212.825,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 1.147009377926588,
|
|
"epoch": 0.3447641812999908,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.11257202178239822,
|
|
"kl": 0.039408953103702515,
|
|
"learning_rate": 4.942682192991024e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 16228631.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.42775094211101533,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.4277509480714798,
|
|
"sampling/importance_sampling_ratio/max": 1.3021114349365235,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002467393875123,
|
|
"sampling/importance_sampling_ratio/min": 0.7513287425041199,
|
|
"sampling/sampling_logp_difference/max": 0.3089804947376251,
|
|
"sampling/sampling_logp_difference/mean": 0.022145429626107216,
|
|
"step": 3750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.4,
|
|
"completions/max_terminated_length": 270.4,
|
|
"completions/mean_length": 209.025,
|
|
"completions/mean_terminated_length": 209.025,
|
|
"completions/min_length": 143.1,
|
|
"completions/min_terminated_length": 143.1,
|
|
"entropy": 1.04922189489007,
|
|
"epoch": 0.3456835524501241,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.559751987457275,
|
|
"kl": 0.04193950870539993,
|
|
"learning_rate": 4.942141517090415e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 16272425.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.4817752480506897,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.48177525997161863,
|
|
"sampling/importance_sampling_ratio/max": 1.2497560858726502,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995654940605163,
|
|
"sampling/importance_sampling_ratio/min": 0.7870202004909516,
|
|
"sampling/sampling_logp_difference/max": 0.2617586374282837,
|
|
"sampling/sampling_logp_difference/mean": 0.020650873146951198,
|
|
"step": 3760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.4,
|
|
"completions/max_terminated_length": 258.4,
|
|
"completions/mean_length": 193.4,
|
|
"completions/mean_terminated_length": 193.4,
|
|
"completions/min_length": 129.5,
|
|
"completions/min_terminated_length": 129.5,
|
|
"entropy": 1.0805205300450325,
|
|
"epoch": 0.3466029236002574,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.2005414962768555,
|
|
"kl": 0.03449253172148019,
|
|
"learning_rate": 4.941598332914707e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 16315489.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.30355427265167234,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.30355427861213685,
|
|
"sampling/importance_sampling_ratio/max": 1.2482670187950133,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004632771015167,
|
|
"sampling/importance_sampling_ratio/min": 0.7893210172653198,
|
|
"sampling/sampling_logp_difference/max": 0.25456771850585935,
|
|
"sampling/sampling_logp_difference/mean": 0.021257933415472507,
|
|
"step": 3770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.1,
|
|
"completions/max_terminated_length": 261.1,
|
|
"completions/mean_length": 197.025,
|
|
"completions/mean_terminated_length": 197.025,
|
|
"completions/min_length": 133.0,
|
|
"completions/min_terminated_length": 133.0,
|
|
"entropy": 1.0283584214746953,
|
|
"epoch": 0.34752229475039076,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.11715030670166,
|
|
"kl": 0.03571356469765306,
|
|
"learning_rate": 4.941052641021793e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 16361083.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.3774814397096634,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.3774814516305923,
|
|
"sampling/importance_sampling_ratio/max": 1.2746479749679565,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005091607570649,
|
|
"sampling/importance_sampling_ratio/min": 0.7983085095882416,
|
|
"sampling/sampling_logp_difference/max": 0.2533171534538269,
|
|
"sampling/sampling_logp_difference/mean": 0.020525658130645753,
|
|
"step": 3780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 293.7,
|
|
"completions/max_terminated_length": 267.5,
|
|
"completions/mean_length": 197.3,
|
|
"completions/mean_terminated_length": 193.1714294433594,
|
|
"completions/min_length": 115.9,
|
|
"completions/min_terminated_length": 115.9,
|
|
"entropy": 1.024768951535225,
|
|
"epoch": 0.34844166590052406,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14785781502723694,
|
|
"kl": 0.038201427122112365,
|
|
"learning_rate": 4.940504441972142e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 16403099.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.3418251842260361,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.34182519614696505,
|
|
"sampling/importance_sampling_ratio/max": 1.2535872340202332,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001335263252258,
|
|
"sampling/importance_sampling_ratio/min": 0.7969481885433197,
|
|
"sampling/sampling_logp_difference/max": 0.25146701335906985,
|
|
"sampling/sampling_logp_difference/mean": 0.020142976008355618,
|
|
"step": 3790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 249.6,
|
|
"completions/max_terminated_length": 249.6,
|
|
"completions/mean_length": 184.9,
|
|
"completions/mean_terminated_length": 184.9,
|
|
"completions/min_length": 124.1,
|
|
"completions/min_terminated_length": 124.1,
|
|
"entropy": 1.0642747826874257,
|
|
"epoch": 0.34936103705065735,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.6343207359313965,
|
|
"kl": 0.040106067794840786,
|
|
"learning_rate": 4.939953736328799e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 16442803.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.2722664296627045,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.2722664475440979,
|
|
"sampling/importance_sampling_ratio/max": 1.2576985359191895,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996882736682892,
|
|
"sampling/importance_sampling_ratio/min": 0.7921940565109253,
|
|
"sampling/sampling_logp_difference/max": 0.24677510261535646,
|
|
"sampling/sampling_logp_difference/mean": 0.021370902471244336,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"epoch": 0.34936103705065735,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002617801047120419,
|
|
"eval_completions/max_length": 288.4013961605585,
|
|
"eval_completions/max_terminated_length": 285.5183246073298,
|
|
"eval_completions/mean_length": 202.68215532286212,
|
|
"eval_completions/mean_terminated_length": 202.03300950081976,
|
|
"eval_completions/min_length": 131.21640488656195,
|
|
"eval_completions/min_terminated_length": 131.21640488656195,
|
|
"eval_entropy": 1.0889998854870988,
|
|
"eval_frac_reward_zero_std": 0.2914485165794066,
|
|
"eval_kl": 0.03765272300033773,
|
|
"eval_loss": 0.0019535778556019068,
|
|
"eval_num_tokens": 16442803.0,
|
|
"eval_reward": 0.04308464223385689,
|
|
"eval_reward_std": 0.37965829766649645,
|
|
"eval_rewards/reward_fn/mean": 0.04308464223385689,
|
|
"eval_rewards/reward_fn/std": 0.37965830796468114,
|
|
"eval_runtime": 1689.934,
|
|
"eval_samples_per_second": 0.339,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2654195166799203,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999659575300899,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7853988346629118,
|
|
"eval_sampling/sampling_logp_difference/max": 0.26999440357531135,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020895753397133336,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.9,
|
|
"completions/max_terminated_length": 287.9,
|
|
"completions/mean_length": 195.1875,
|
|
"completions/mean_terminated_length": 195.1875,
|
|
"completions/min_length": 113.8,
|
|
"completions/min_terminated_length": 113.8,
|
|
"entropy": 1.0213661059737205,
|
|
"epoch": 0.35028040820079065,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.19238805770874,
|
|
"kl": 0.04155476473970339,
|
|
"learning_rate": 4.9394005246573826e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 16485538.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.5410771846771241,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.541077196598053,
|
|
"sampling/importance_sampling_ratio/max": 1.2793991088867187,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003066837787629,
|
|
"sampling/importance_sampling_ratio/min": 0.8043634951114654,
|
|
"sampling/sampling_logp_difference/max": 0.24925187826156617,
|
|
"sampling/sampling_logp_difference/mean": 0.02093137949705124,
|
|
"step": 3810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.3,
|
|
"completions/max_terminated_length": 298.3,
|
|
"completions/mean_length": 211.2375,
|
|
"completions/mean_terminated_length": 211.2375,
|
|
"completions/min_length": 123.2,
|
|
"completions/min_terminated_length": 123.2,
|
|
"entropy": 1.07375026345253,
|
|
"epoch": 0.35119977935092395,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1306552141904831,
|
|
"kl": 0.030958269751863554,
|
|
"learning_rate": 4.938844807526085e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 16527917.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.40066249668598175,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.4006625026464462,
|
|
"sampling/importance_sampling_ratio/max": 1.2238177061080933,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003024578094482,
|
|
"sampling/importance_sampling_ratio/min": 0.7825372815132141,
|
|
"sampling/sampling_logp_difference/max": 0.24665732383728028,
|
|
"sampling/sampling_logp_difference/mean": 0.02107987031340599,
|
|
"step": 3820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.2,
|
|
"completions/max_terminated_length": 295.2,
|
|
"completions/mean_length": 217.875,
|
|
"completions/mean_terminated_length": 217.875,
|
|
"completions/min_length": 141.2,
|
|
"completions/min_terminated_length": 141.2,
|
|
"entropy": 1.0477053500711917,
|
|
"epoch": 0.3521191505010573,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.669813632965088,
|
|
"kl": 0.03516795663163066,
|
|
"learning_rate": 4.938286585505673e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 16573587.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3466688096523285,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.3466688334941864,
|
|
"sampling/importance_sampling_ratio/max": 1.2880855798721313,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997962713241577,
|
|
"sampling/importance_sampling_ratio/min": 0.7429078757762909,
|
|
"sampling/sampling_logp_difference/max": 0.32917438745498656,
|
|
"sampling/sampling_logp_difference/mean": 0.020702942460775375,
|
|
"step": 3830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 256.6,
|
|
"completions/max_terminated_length": 256.6,
|
|
"completions/mean_length": 191.6125,
|
|
"completions/mean_terminated_length": 191.6125,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"entropy": 1.0914020456373692,
|
|
"epoch": 0.3530385216511906,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.068206787109375,
|
|
"kl": 0.04721885845065117,
|
|
"learning_rate": 4.9377258591694854e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 16615004.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.47980677485466006,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.47980678677558897,
|
|
"sampling/importance_sampling_ratio/max": 1.3143089056015014,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000809967517852,
|
|
"sampling/importance_sampling_ratio/min": 0.771519011259079,
|
|
"sampling/sampling_logp_difference/max": 0.30351110100746154,
|
|
"sampling/sampling_logp_difference/mean": 0.02236689832061529,
|
|
"step": 3840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.0,
|
|
"completions/max_terminated_length": 276.0,
|
|
"completions/mean_length": 205.375,
|
|
"completions/mean_terminated_length": 205.375,
|
|
"completions/min_length": 126.7,
|
|
"completions/min_terminated_length": 126.7,
|
|
"entropy": 1.0853016667068005,
|
|
"epoch": 0.3539578928013239,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.1450071781873703,
|
|
"kl": 0.03386314922827296,
|
|
"learning_rate": 4.937162629093433e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 16657802.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.4728062242269516,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.4728062301874161,
|
|
"sampling/importance_sampling_ratio/max": 1.2426780104637145,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004954636096954,
|
|
"sampling/importance_sampling_ratio/min": 0.7994471788406372,
|
|
"sampling/sampling_logp_difference/max": 0.24012725353240966,
|
|
"sampling/sampling_logp_difference/mean": 0.021801168285310268,
|
|
"step": 3850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.3,
|
|
"completions/max_terminated_length": 309.3,
|
|
"completions/mean_length": 219.8875,
|
|
"completions/mean_terminated_length": 219.8875,
|
|
"completions/min_length": 128.9,
|
|
"completions/min_terminated_length": 128.9,
|
|
"entropy": 1.1077279642224311,
|
|
"epoch": 0.3548772639514572,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.3538594245910645,
|
|
"kl": 0.03275170275010168,
|
|
"learning_rate": 4.936596895855998e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 16702505.0,
|
|
"reward": -0.35,
|
|
"reward_std": 0.40359844267368317,
|
|
"rewards/reward_fn/mean": -0.35,
|
|
"rewards/reward_fn/std": 0.4035984486341476,
|
|
"sampling/importance_sampling_ratio/max": 1.2595935821533204,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996255397796631,
|
|
"sampling/importance_sampling_ratio/min": 0.7407901108264923,
|
|
"sampling/sampling_logp_difference/max": 0.3108835220336914,
|
|
"sampling/sampling_logp_difference/mean": 0.022558890655636786,
|
|
"step": 3860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.0,
|
|
"completions/max_terminated_length": 316.0,
|
|
"completions/mean_length": 227.6375,
|
|
"completions/mean_terminated_length": 227.6375,
|
|
"completions/min_length": 142.4,
|
|
"completions/min_terminated_length": 142.4,
|
|
"entropy": 1.0208419144153595,
|
|
"epoch": 0.3557966351015905,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.20402826368808746,
|
|
"kl": 0.033885483851190654,
|
|
"learning_rate": 4.9360286600382326e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 16746148.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4000727891921997,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.40007280707359316,
|
|
"sampling/importance_sampling_ratio/max": 1.246892273426056,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000983595848083,
|
|
"sampling/importance_sampling_ratio/min": 0.801948344707489,
|
|
"sampling/sampling_logp_difference/max": 0.23523476123809814,
|
|
"sampling/sampling_logp_difference/mean": 0.020918942429125308,
|
|
"step": 3870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.5,
|
|
"completions/max_terminated_length": 324.5,
|
|
"completions/mean_length": 232.6125,
|
|
"completions/mean_terminated_length": 232.6125,
|
|
"completions/min_length": 151.7,
|
|
"completions/min_terminated_length": 151.7,
|
|
"entropy": 1.0896278016269207,
|
|
"epoch": 0.35671600625172384,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.469940662384033,
|
|
"kl": 0.029803099611308427,
|
|
"learning_rate": 4.9354579222237624e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 16793605.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.3946571469306946,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.394657164812088,
|
|
"sampling/importance_sampling_ratio/max": 1.283231246471405,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998651146888733,
|
|
"sampling/importance_sampling_ratio/min": 0.7650677621364593,
|
|
"sampling/sampling_logp_difference/max": 0.2906580984592438,
|
|
"sampling/sampling_logp_difference/mean": 0.021508488245308398,
|
|
"step": 3880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 323.8,
|
|
"completions/max_terminated_length": 306.3,
|
|
"completions/mean_length": 220.75,
|
|
"completions/mean_terminated_length": 217.78035888671874,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 1.1108997106552123,
|
|
"epoch": 0.35763537740185714,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.163994789123535,
|
|
"kl": 0.03250877143582329,
|
|
"learning_rate": 4.934884682998781e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 16837097.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.44252554178237913,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.44252554178237913,
|
|
"sampling/importance_sampling_ratio/max": 1.3308586359024048,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002409398555756,
|
|
"sampling/importance_sampling_ratio/min": 0.7470481276512146,
|
|
"sampling/sampling_logp_difference/max": 0.32441638708114623,
|
|
"sampling/sampling_logp_difference/mean": 0.021972625330090523,
|
|
"step": 3890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 346.0,
|
|
"completions/max_terminated_length": 331.4,
|
|
"completions/mean_length": 232.9,
|
|
"completions/mean_terminated_length": 229.64642944335938,
|
|
"completions/min_length": 155.2,
|
|
"completions/min_terminated_length": 155.2,
|
|
"entropy": 1.0732648648321628,
|
|
"epoch": 0.35855474855199043,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.840244770050049,
|
|
"kl": 0.02413992037763819,
|
|
"learning_rate": 4.934308942952049e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 16882065.0,
|
|
"reward": 0.06875,
|
|
"reward_std": 0.4280139446258545,
|
|
"rewards/reward_fn/mean": 0.06875,
|
|
"rewards/reward_fn/std": 0.42801395058631897,
|
|
"sampling/importance_sampling_ratio/max": 1.2515279054641724,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998293399810791,
|
|
"sampling/importance_sampling_ratio/min": 0.7841599225997925,
|
|
"sampling/sampling_logp_difference/max": 0.25341911911964415,
|
|
"sampling/sampling_logp_difference/mean": 0.02119055576622486,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.0,
|
|
"completions/max_terminated_length": 276.0,
|
|
"completions/mean_length": 202.475,
|
|
"completions/mean_terminated_length": 202.475,
|
|
"completions/min_length": 139.2,
|
|
"completions/min_terminated_length": 139.2,
|
|
"entropy": 1.0567621059715748,
|
|
"epoch": 0.35947411970212373,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.396238803863525,
|
|
"kl": 0.024067029828438537,
|
|
"learning_rate": 4.933730702674899e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 16926479.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.20580926835536956,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.20580926835536956,
|
|
"sampling/importance_sampling_ratio/max": 1.2392575860023498,
|
|
"sampling/importance_sampling_ratio/mean": 1.000095546245575,
|
|
"sampling/importance_sampling_ratio/min": 0.7632444858551025,
|
|
"sampling/sampling_logp_difference/max": 0.2875574231147766,
|
|
"sampling/sampling_logp_difference/mean": 0.020880523882806302,
|
|
"step": 3910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 266.7,
|
|
"completions/max_terminated_length": 266.7,
|
|
"completions/mean_length": 197.0875,
|
|
"completions/mean_terminated_length": 197.0875,
|
|
"completions/min_length": 136.3,
|
|
"completions/min_terminated_length": 136.3,
|
|
"entropy": 1.0886995639652013,
|
|
"epoch": 0.3603934908522571,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.988309383392334,
|
|
"kl": 0.030136342253535987,
|
|
"learning_rate": 4.933149962761231e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 16968718.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.5064159631729126,
|
|
"rewards/reward_fn/mean": 0.45,
|
|
"rewards/reward_fn/std": 0.506415969133377,
|
|
"sampling/importance_sampling_ratio/max": 1.2369025349617004,
|
|
"sampling/importance_sampling_ratio/mean": 1.000149667263031,
|
|
"sampling/importance_sampling_ratio/min": 0.7756860613822937,
|
|
"sampling/sampling_logp_difference/max": 0.26561460494995115,
|
|
"sampling/sampling_logp_difference/mean": 0.02201034687459469,
|
|
"step": 3920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.1,
|
|
"completions/max_terminated_length": 298.1,
|
|
"completions/mean_length": 204.3375,
|
|
"completions/mean_terminated_length": 204.3375,
|
|
"completions/min_length": 132.9,
|
|
"completions/min_terminated_length": 132.9,
|
|
"entropy": 1.031305456161499,
|
|
"epoch": 0.3613128620023904,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.9556803703308105,
|
|
"kl": 0.02554867728613317,
|
|
"learning_rate": 4.93256672380751e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 17012417.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.5804221421480179,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.5804221600294113,
|
|
"sampling/importance_sampling_ratio/max": 1.3038613438606261,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000344038009643,
|
|
"sampling/importance_sampling_ratio/min": 0.7963452279567719,
|
|
"sampling/sampling_logp_difference/max": 0.2828441858291626,
|
|
"sampling/sampling_logp_difference/mean": 0.020876816473901272,
|
|
"step": 3930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.7,
|
|
"completions/max_terminated_length": 255.7,
|
|
"completions/mean_length": 183.275,
|
|
"completions/mean_terminated_length": 183.275,
|
|
"completions/min_length": 111.5,
|
|
"completions/min_terminated_length": 111.5,
|
|
"entropy": 1.05137086994946,
|
|
"epoch": 0.3622322331525237,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.099802017211914,
|
|
"kl": 0.03895709650823846,
|
|
"learning_rate": 4.931980986412769e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 17053023.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.5395444571971894,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.5395444750785827,
|
|
"sampling/importance_sampling_ratio/max": 1.2688334822654723,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002437829971313,
|
|
"sampling/importance_sampling_ratio/min": 0.7317319363355637,
|
|
"sampling/sampling_logp_difference/max": 0.3511121034622192,
|
|
"sampling/sampling_logp_difference/mean": 0.021704531740397215,
|
|
"step": 3940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.2,
|
|
"completions/max_terminated_length": 294.2,
|
|
"completions/mean_length": 219.8875,
|
|
"completions/mean_terminated_length": 219.8875,
|
|
"completions/min_length": 140.1,
|
|
"completions/min_terminated_length": 140.1,
|
|
"entropy": 1.1323226742446422,
|
|
"epoch": 0.36315160430265697,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.40850305557251,
|
|
"kl": 0.028351474611554295,
|
|
"learning_rate": 4.931392751178608e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 17100030.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.421695351600647,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.42169535756111143,
|
|
"sampling/importance_sampling_ratio/max": 1.2507598400115967,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999391674995423,
|
|
"sampling/importance_sampling_ratio/min": 0.7372199535369873,
|
|
"sampling/sampling_logp_difference/max": 0.33733655512332916,
|
|
"sampling/sampling_logp_difference/mean": 0.021574048697948454,
|
|
"step": 3950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.0,
|
|
"completions/max_terminated_length": 325.0,
|
|
"completions/mean_length": 228.9375,
|
|
"completions/mean_terminated_length": 228.9375,
|
|
"completions/min_length": 151.1,
|
|
"completions/min_terminated_length": 151.1,
|
|
"entropy": 1.1639699287712575,
|
|
"epoch": 0.36407097545279027,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.5508328676223755,
|
|
"kl": 0.02543405604083091,
|
|
"learning_rate": 4.930802018709192e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 17144409.0,
|
|
"reward": -0.11875,
|
|
"reward_std": 0.4696974903345108,
|
|
"rewards/reward_fn/mean": -0.11875,
|
|
"rewards/reward_fn/std": 0.46969751417636874,
|
|
"sampling/importance_sampling_ratio/max": 1.3100485801696777,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001014530658723,
|
|
"sampling/importance_sampling_ratio/min": 0.7754747748374939,
|
|
"sampling/sampling_logp_difference/max": 0.3061722874641418,
|
|
"sampling/sampling_logp_difference/mean": 0.021867893263697626,
|
|
"step": 3960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.6,
|
|
"completions/max_terminated_length": 269.6,
|
|
"completions/mean_length": 205.8875,
|
|
"completions/mean_terminated_length": 205.8875,
|
|
"completions/min_length": 132.7,
|
|
"completions/min_terminated_length": 132.7,
|
|
"entropy": 1.0047426655888558,
|
|
"epoch": 0.3649903466029236,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11208415031433105,
|
|
"kl": 0.022424453136045484,
|
|
"learning_rate": 4.9302087896112506e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 17186288.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.2857582807540894,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.28575828671455383,
|
|
"sampling/importance_sampling_ratio/max": 1.2722222805023193,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997186779975891,
|
|
"sampling/importance_sampling_ratio/min": 0.7932699680328369,
|
|
"sampling/sampling_logp_difference/max": 0.2713023662567139,
|
|
"sampling/sampling_logp_difference/mean": 0.020550216920673846,
|
|
"step": 3970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.5,
|
|
"completions/max_terminated_length": 289.5,
|
|
"completions/mean_length": 196.1125,
|
|
"completions/mean_terminated_length": 196.1125,
|
|
"completions/min_length": 131.7,
|
|
"completions/min_terminated_length": 131.7,
|
|
"entropy": 1.0293453693389893,
|
|
"epoch": 0.3659097177530569,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.847952842712402,
|
|
"kl": 0.028373274678597227,
|
|
"learning_rate": 4.929613064494076e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 17225521.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.4863525778055191,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.48635258376598356,
|
|
"sampling/importance_sampling_ratio/max": 1.330295491218567,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002269804477693,
|
|
"sampling/importance_sampling_ratio/min": 0.7863039195537567,
|
|
"sampling/sampling_logp_difference/max": 0.3064713627099991,
|
|
"sampling/sampling_logp_difference/mean": 0.020670720376074314,
|
|
"step": 3980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.8,
|
|
"completions/max_terminated_length": 288.8,
|
|
"completions/mean_length": 202.875,
|
|
"completions/mean_terminated_length": 202.875,
|
|
"completions/min_length": 120.8,
|
|
"completions/min_terminated_length": 120.8,
|
|
"entropy": 1.024110032618046,
|
|
"epoch": 0.3668290889031902,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.5246806144714355,
|
|
"kl": 0.022712926811072974,
|
|
"learning_rate": 4.929014843969527e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 17266399.0,
|
|
"reward": 0.19375,
|
|
"reward_std": 0.35878387838602066,
|
|
"rewards/reward_fn/mean": 0.19375,
|
|
"rewards/reward_fn/std": 0.35878387838602066,
|
|
"sampling/importance_sampling_ratio/max": 1.2430195927619934,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006589531898498,
|
|
"sampling/importance_sampling_ratio/min": 0.7966007173061371,
|
|
"sampling/sampling_logp_difference/max": 0.2463855743408203,
|
|
"sampling/sampling_logp_difference/mean": 0.02132015787065029,
|
|
"step": 3990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.6,
|
|
"completions/max_terminated_length": 272.6,
|
|
"completions/mean_length": 190.9,
|
|
"completions/mean_terminated_length": 190.9,
|
|
"completions/min_length": 114.6,
|
|
"completions/min_terminated_length": 114.6,
|
|
"entropy": 1.0733345195651054,
|
|
"epoch": 0.3677484600533235,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.040745735168457,
|
|
"kl": 0.023259436117950828,
|
|
"learning_rate": 4.9284141286520245e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 17308143.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.4148233026266098,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.41482333838939667,
|
|
"sampling/importance_sampling_ratio/max": 1.3117130279541016,
|
|
"sampling/importance_sampling_ratio/mean": 1.000256496667862,
|
|
"sampling/importance_sampling_ratio/min": 0.792364090681076,
|
|
"sampling/sampling_logp_difference/max": 0.2844870686531067,
|
|
"sampling/sampling_logp_difference/mean": 0.021840430423617364,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.3677484600533235,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0015270506108202443,
|
|
"eval_completions/max_length": 283.88132635253055,
|
|
"eval_completions/max_terminated_length": 282.68411867364745,
|
|
"eval_completions/mean_length": 201.9705497382199,
|
|
"eval_completions/mean_terminated_length": 201.58239844623452,
|
|
"eval_completions/min_length": 132.88132635253055,
|
|
"eval_completions/min_terminated_length": 132.88132635253055,
|
|
"eval_entropy": 1.0520488543348163,
|
|
"eval_frac_reward_zero_std": 0.29668411867364747,
|
|
"eval_kl": 0.030604112914472455,
|
|
"eval_loss": 0.001616702415049076,
|
|
"eval_num_tokens": 17308143.0,
|
|
"eval_reward": 0.031195462478184992,
|
|
"eval_reward_std": 0.40771024956337026,
|
|
"eval_rewards/reward_fn/mean": 0.031195462478184992,
|
|
"eval_rewards/reward_fn/std": 0.4077102613178639,
|
|
"eval_runtime": 1666.3235,
|
|
"eval_samples_per_second": 0.344,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.271846962848883,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999645783014947,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7824295284340311,
|
|
"eval_sampling/sampling_logp_difference/max": 0.273646653660721,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020698034389831945,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 280.5,
|
|
"completions/max_terminated_length": 264.5,
|
|
"completions/mean_length": 204.2375,
|
|
"completions/mean_terminated_length": 201.2517883300781,
|
|
"completions/min_length": 137.4,
|
|
"completions/min_terminated_length": 137.4,
|
|
"entropy": 0.9521341491490602,
|
|
"epoch": 0.3686678312034568,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.164662167429924,
|
|
"kl": 0.02677487238543108,
|
|
"learning_rate": 4.92781091915855e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 17349866.0,
|
|
"reward": 0.23125,
|
|
"reward_std": 0.44940980672836306,
|
|
"rewards/reward_fn/mean": 0.23125,
|
|
"rewards/reward_fn/std": 0.4494098246097565,
|
|
"sampling/importance_sampling_ratio/max": 1.2505493760108948,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997177422046661,
|
|
"sampling/importance_sampling_ratio/min": 0.7825440526008606,
|
|
"sampling/sampling_logp_difference/max": 0.25838118195533755,
|
|
"sampling/sampling_logp_difference/mean": 0.019815102312713863,
|
|
"step": 4010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 323.5,
|
|
"completions/max_terminated_length": 323.5,
|
|
"completions/mean_length": 230.4625,
|
|
"completions/mean_terminated_length": 230.4625,
|
|
"completions/min_length": 154.1,
|
|
"completions/min_terminated_length": 154.1,
|
|
"entropy": 1.0772514939308167,
|
|
"epoch": 0.36958720235359016,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.355861186981201,
|
|
"kl": 0.030957560474053025,
|
|
"learning_rate": 4.927205216108649e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 17396319.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.35137554705142976,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.3513755530118942,
|
|
"sampling/importance_sampling_ratio/max": 1.2867622375488281,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999241530895233,
|
|
"sampling/importance_sampling_ratio/min": 0.7290894627571106,
|
|
"sampling/sampling_logp_difference/max": 0.33557829856872556,
|
|
"sampling/sampling_logp_difference/mean": 0.021585827134549616,
|
|
"step": 4020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.9,
|
|
"completions/max_terminated_length": 313.9,
|
|
"completions/mean_length": 238.8875,
|
|
"completions/mean_terminated_length": 238.8875,
|
|
"completions/min_length": 167.3,
|
|
"completions/min_terminated_length": 167.3,
|
|
"entropy": 1.1207863569259644,
|
|
"epoch": 0.37050657350372346,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.0623884201049805,
|
|
"kl": 0.022352238243911415,
|
|
"learning_rate": 4.926597020124428e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 17445030.0,
|
|
"reward": -0.45,
|
|
"reward_std": 0.28049272000789643,
|
|
"rewards/reward_fn/mean": -0.45,
|
|
"rewards/reward_fn/std": 0.28049272000789643,
|
|
"sampling/importance_sampling_ratio/max": 1.2419703960418702,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994587421417236,
|
|
"sampling/importance_sampling_ratio/min": 0.7675145030021667,
|
|
"sampling/sampling_logp_difference/max": 0.27184714674949645,
|
|
"sampling/sampling_logp_difference/mean": 0.021614385396242143,
|
|
"step": 4030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 304.9,
|
|
"completions/max_terminated_length": 295.8,
|
|
"completions/mean_length": 217.925,
|
|
"completions/mean_terminated_length": 215.00357360839843,
|
|
"completions/min_length": 140.3,
|
|
"completions/min_terminated_length": 140.3,
|
|
"entropy": 0.9822823759168386,
|
|
"epoch": 0.37142594465385675,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.078151226043701,
|
|
"kl": 0.023352814401732758,
|
|
"learning_rate": 4.92598633183055e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 17485512.0,
|
|
"reward": -0.01875,
|
|
"reward_std": 0.3300223082304001,
|
|
"rewards/reward_fn/mean": -0.01875,
|
|
"rewards/reward_fn/std": 0.3300223141908646,
|
|
"sampling/importance_sampling_ratio/max": 1.262546670436859,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999734222888946,
|
|
"sampling/importance_sampling_ratio/min": 0.7684449434280396,
|
|
"sampling/sampling_logp_difference/max": 0.3010485053062439,
|
|
"sampling/sampling_logp_difference/mean": 0.019567642360925674,
|
|
"step": 4040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 327.6,
|
|
"completions/max_terminated_length": 318.1,
|
|
"completions/mean_length": 222.1375,
|
|
"completions/mean_terminated_length": 220.26964416503907,
|
|
"completions/min_length": 147.0,
|
|
"completions/min_terminated_length": 147.0,
|
|
"entropy": 1.093243619799614,
|
|
"epoch": 0.37234531580399005,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 9.030506134033203,
|
|
"kl": 0.022978481504833326,
|
|
"learning_rate": 4.925373151854244e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 17528347.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.5721349716186523,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.5721349954605103,
|
|
"sampling/importance_sampling_ratio/max": 1.2676013708114624,
|
|
"sampling/importance_sampling_ratio/mean": 1.000027734041214,
|
|
"sampling/importance_sampling_ratio/min": 0.7944826364517212,
|
|
"sampling/sampling_logp_difference/max": 0.2545661449432373,
|
|
"sampling/sampling_logp_difference/mean": 0.021137323044240475,
|
|
"step": 4050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.1,
|
|
"completions/max_terminated_length": 284.1,
|
|
"completions/mean_length": 211.1625,
|
|
"completions/mean_terminated_length": 211.1625,
|
|
"completions/min_length": 128.1,
|
|
"completions/min_terminated_length": 128.1,
|
|
"entropy": 1.0667280025780201,
|
|
"epoch": 0.3732646869541234,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14511102437973022,
|
|
"kl": 0.031603105168323965,
|
|
"learning_rate": 4.924757480825292e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 17569384.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.36135117411613465,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.3613511800765991,
|
|
"sampling/importance_sampling_ratio/max": 1.2820291757583617,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999892592430115,
|
|
"sampling/importance_sampling_ratio/min": 0.7780540645122528,
|
|
"sampling/sampling_logp_difference/max": 0.27045043706893923,
|
|
"sampling/sampling_logp_difference/mean": 0.021450281143188477,
|
|
"step": 4060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.3,
|
|
"completions/max_terminated_length": 309.3,
|
|
"completions/mean_length": 215.65,
|
|
"completions/mean_terminated_length": 215.65,
|
|
"completions/min_length": 143.4,
|
|
"completions/min_terminated_length": 143.4,
|
|
"entropy": 1.01245990768075,
|
|
"epoch": 0.3741840581042567,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.895285606384277,
|
|
"kl": 0.030177100194850936,
|
|
"learning_rate": 4.92413931937604e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 17614092.0,
|
|
"reward": -0.3125,
|
|
"reward_std": 0.43944525718688965,
|
|
"rewards/reward_fn/mean": -0.3125,
|
|
"rewards/reward_fn/std": 0.43944527506828307,
|
|
"sampling/importance_sampling_ratio/max": 1.2394626140594482,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996505796909332,
|
|
"sampling/importance_sampling_ratio/min": 0.7653551936149597,
|
|
"sampling/sampling_logp_difference/max": 0.27353883981704713,
|
|
"sampling/sampling_logp_difference/mean": 0.019894703291356564,
|
|
"step": 4070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.9,
|
|
"completions/max_terminated_length": 303.9,
|
|
"completions/mean_length": 233.2125,
|
|
"completions/mean_terminated_length": 233.2125,
|
|
"completions/min_length": 157.3,
|
|
"completions/min_terminated_length": 157.3,
|
|
"entropy": 1.1959784880280495,
|
|
"epoch": 0.37510342925439,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.870891094207764,
|
|
"kl": 0.03600883090402931,
|
|
"learning_rate": 4.923518668141388e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 17660477.0,
|
|
"reward": -0.325,
|
|
"reward_std": 0.4479423314332962,
|
|
"rewards/reward_fn/mean": -0.325,
|
|
"rewards/reward_fn/std": 0.44794235527515414,
|
|
"sampling/importance_sampling_ratio/max": 1.3080520272254943,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002026081085205,
|
|
"sampling/importance_sampling_ratio/min": 0.7897085726261139,
|
|
"sampling/sampling_logp_difference/max": 0.29439462423324586,
|
|
"sampling/sampling_logp_difference/mean": 0.022370431944727896,
|
|
"step": 4080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.2,
|
|
"completions/max_terminated_length": 299.2,
|
|
"completions/mean_length": 225.125,
|
|
"completions/mean_terminated_length": 225.125,
|
|
"completions/min_length": 158.9,
|
|
"completions/min_terminated_length": 158.9,
|
|
"entropy": 1.1307223968207836,
|
|
"epoch": 0.3760228004045233,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.4964213371276855,
|
|
"kl": 0.029724794812500475,
|
|
"learning_rate": 4.922895527758795e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 17705735.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.43609668016433717,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.4360966980457306,
|
|
"sampling/importance_sampling_ratio/max": 1.2420305728912353,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001307249069213,
|
|
"sampling/importance_sampling_ratio/min": 0.7889919281005859,
|
|
"sampling/sampling_logp_difference/max": 0.25029834508895876,
|
|
"sampling/sampling_logp_difference/mean": 0.021047768741846086,
|
|
"step": 4090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.2,
|
|
"completions/max_terminated_length": 287.2,
|
|
"completions/mean_length": 206.3875,
|
|
"completions/mean_terminated_length": 206.3875,
|
|
"completions/min_length": 131.3,
|
|
"completions/min_terminated_length": 131.3,
|
|
"entropy": 1.1340158827602864,
|
|
"epoch": 0.3769421715546566,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.3849077224731445,
|
|
"kl": 0.038168801891151814,
|
|
"learning_rate": 4.9222698988682775e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 17748926.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.44958853125572207,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.4495885372161865,
|
|
"sampling/importance_sampling_ratio/max": 1.2284218430519105,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005335867404939,
|
|
"sampling/importance_sampling_ratio/min": 0.7881789267063141,
|
|
"sampling/sampling_logp_difference/max": 0.2432108163833618,
|
|
"sampling/sampling_logp_difference/mean": 0.021849458292126655,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 294.9,
|
|
"completions/max_terminated_length": 272.9,
|
|
"completions/mean_length": 200.05,
|
|
"completions/mean_terminated_length": 196.09285736083984,
|
|
"completions/min_length": 135.0,
|
|
"completions/min_terminated_length": 135.0,
|
|
"entropy": 1.069764541834593,
|
|
"epoch": 0.37786154270478994,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.13810424506664276,
|
|
"kl": 0.04081633408786729,
|
|
"learning_rate": 4.921641782112404e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 17791706.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4460426539182663,
|
|
"rewards/reward_fn/mean": 0.15625,
|
|
"rewards/reward_fn/std": 0.44604267179965973,
|
|
"sampling/importance_sampling_ratio/max": 1.262696921825409,
|
|
"sampling/importance_sampling_ratio/mean": 0.999956876039505,
|
|
"sampling/importance_sampling_ratio/min": 0.7846767902374268,
|
|
"sampling/sampling_logp_difference/max": 0.27911136150360105,
|
|
"sampling/sampling_logp_difference/mean": 0.021073807775974274,
|
|
"step": 4110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.3,
|
|
"completions/max_terminated_length": 288.3,
|
|
"completions/mean_length": 207.5375,
|
|
"completions/mean_terminated_length": 207.5375,
|
|
"completions/min_length": 148.2,
|
|
"completions/min_terminated_length": 148.2,
|
|
"entropy": 1.026875903457403,
|
|
"epoch": 0.37878091385492324,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.1792694628238678,
|
|
"kl": 0.0306867724400945,
|
|
"learning_rate": 4.921011178136302e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 17834725.0,
|
|
"reward": 0.5875,
|
|
"reward_std": 0.24493903219699859,
|
|
"rewards/reward_fn/mean": 0.5875,
|
|
"rewards/reward_fn/std": 0.24493903219699859,
|
|
"sampling/importance_sampling_ratio/max": 1.2497349858283997,
|
|
"sampling/importance_sampling_ratio/mean": 1.000208204984665,
|
|
"sampling/importance_sampling_ratio/min": 0.7569918751716613,
|
|
"sampling/sampling_logp_difference/max": 0.2868160128593445,
|
|
"sampling/sampling_logp_difference/mean": 0.020739179104566574,
|
|
"step": 4120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 323.6,
|
|
"completions/max_terminated_length": 323.6,
|
|
"completions/mean_length": 232.8625,
|
|
"completions/mean_terminated_length": 232.8625,
|
|
"completions/min_length": 151.4,
|
|
"completions/min_terminated_length": 151.4,
|
|
"entropy": 1.086667349934578,
|
|
"epoch": 0.37970028500505654,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.09945480525493622,
|
|
"kl": 0.03359981171088293,
|
|
"learning_rate": 4.920378087587652e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 17880450.0,
|
|
"reward": -0.2,
|
|
"reward_std": 0.3309396654367447,
|
|
"rewards/reward_fn/mean": -0.2,
|
|
"rewards/reward_fn/std": 0.3309396654367447,
|
|
"sampling/importance_sampling_ratio/max": 1.2475878477096558,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995445549488068,
|
|
"sampling/importance_sampling_ratio/min": 0.7682983756065369,
|
|
"sampling/sampling_logp_difference/max": 0.2868174374103546,
|
|
"sampling/sampling_logp_difference/mean": 0.021320104971528052,
|
|
"step": 4130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 338.4,
|
|
"completions/max_terminated_length": 338.4,
|
|
"completions/mean_length": 229.925,
|
|
"completions/mean_terminated_length": 229.925,
|
|
"completions/min_length": 146.6,
|
|
"completions/min_terminated_length": 146.6,
|
|
"entropy": 1.115572091192007,
|
|
"epoch": 0.38061965615518983,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.636646747589111,
|
|
"kl": 0.040472197683993724,
|
|
"learning_rate": 4.91974251111669e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 17924724.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.5299923837184906,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.529992401599884,
|
|
"sampling/importance_sampling_ratio/max": 1.2865466952323914,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998015522956848,
|
|
"sampling/importance_sampling_ratio/min": 0.7645593822002411,
|
|
"sampling/sampling_logp_difference/max": 0.2887804269790649,
|
|
"sampling/sampling_logp_difference/mean": 0.021381690725684167,
|
|
"step": 4140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.9,
|
|
"completions/max_terminated_length": 273.9,
|
|
"completions/mean_length": 199.825,
|
|
"completions/mean_terminated_length": 199.825,
|
|
"completions/min_length": 134.5,
|
|
"completions/min_terminated_length": 134.5,
|
|
"entropy": 1.1013034857809543,
|
|
"epoch": 0.3815390273053232,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.736066818237305,
|
|
"kl": 0.04292634167941287,
|
|
"learning_rate": 4.919104449376202e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 17966302.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.26036913990974425,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.26036914587020876,
|
|
"sampling/importance_sampling_ratio/max": 1.2266042232513428,
|
|
"sampling/importance_sampling_ratio/mean": 0.99991774559021,
|
|
"sampling/importance_sampling_ratio/min": 0.7998579144477844,
|
|
"sampling/sampling_logp_difference/max": 0.2290733575820923,
|
|
"sampling/sampling_logp_difference/mean": 0.021233107335865496,
|
|
"step": 4150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.1,
|
|
"completions/max_terminated_length": 291.1,
|
|
"completions/mean_length": 214.05,
|
|
"completions/mean_terminated_length": 214.05,
|
|
"completions/min_length": 139.3,
|
|
"completions/min_terminated_length": 139.3,
|
|
"entropy": 1.0977688491344453,
|
|
"epoch": 0.3824583984554565,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.757287979125977,
|
|
"kl": 0.03785655811661855,
|
|
"learning_rate": 4.9184639030215295e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 18011130.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.5383017629384994,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.5383017748594284,
|
|
"sampling/importance_sampling_ratio/max": 1.2806958079338073,
|
|
"sampling/importance_sampling_ratio/mean": 1.00012549161911,
|
|
"sampling/importance_sampling_ratio/min": 0.7377934008836746,
|
|
"sampling/sampling_logp_difference/max": 0.32518887519836426,
|
|
"sampling/sampling_logp_difference/mean": 0.021232788637280463,
|
|
"step": 4160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.3,
|
|
"completions/max_terminated_length": 295.3,
|
|
"completions/mean_length": 195.5375,
|
|
"completions/mean_terminated_length": 195.5375,
|
|
"completions/min_length": 116.2,
|
|
"completions/min_terminated_length": 116.2,
|
|
"entropy": 1.1318380936980248,
|
|
"epoch": 0.3833777696055898,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.340631484985352,
|
|
"kl": 0.04119705901248381,
|
|
"learning_rate": 4.917820872710566e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 18054781.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.32111361920833587,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.3211136251688004,
|
|
"sampling/importance_sampling_ratio/max": 1.2262367010116577,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000063061714173,
|
|
"sampling/importance_sampling_ratio/min": 0.7953880190849304,
|
|
"sampling/sampling_logp_difference/max": 0.23239219188690186,
|
|
"sampling/sampling_logp_difference/mean": 0.021214174479246138,
|
|
"step": 4170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 265.7,
|
|
"completions/max_terminated_length": 265.7,
|
|
"completions/mean_length": 186.3625,
|
|
"completions/mean_terminated_length": 186.3625,
|
|
"completions/min_length": 119.5,
|
|
"completions/min_terminated_length": 119.5,
|
|
"entropy": 1.0833267003297806,
|
|
"epoch": 0.3842971407557231,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.464057922363281,
|
|
"kl": 0.05841591847129166,
|
|
"learning_rate": 4.917175359103753e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 18096610.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.43262317180633547,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.4326231896877289,
|
|
"sampling/importance_sampling_ratio/max": 1.2406950116157531,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000700950622559,
|
|
"sampling/importance_sampling_ratio/min": 0.7931158900260925,
|
|
"sampling/sampling_logp_difference/max": 0.24209141731262207,
|
|
"sampling/sampling_logp_difference/mean": 0.02181189451366663,
|
|
"step": 4180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 260.7,
|
|
"completions/max_terminated_length": 260.7,
|
|
"completions/mean_length": 185.875,
|
|
"completions/mean_terminated_length": 185.875,
|
|
"completions/min_length": 123.4,
|
|
"completions/min_terminated_length": 123.4,
|
|
"entropy": 1.0153455756604672,
|
|
"epoch": 0.3852165119058564,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.1384596824646,
|
|
"kl": 0.038847163936588915,
|
|
"learning_rate": 4.9165273628640865e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 18137008.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.49628907442092896,
|
|
"rewards/reward_fn/mean": 0.3875,
|
|
"rewards/reward_fn/std": 0.4962890863418579,
|
|
"sampling/importance_sampling_ratio/max": 1.2498883724212646,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994864761829376,
|
|
"sampling/importance_sampling_ratio/min": 0.7944707155227662,
|
|
"sampling/sampling_logp_difference/max": 0.25554182529449465,
|
|
"sampling/sampling_logp_difference/mean": 0.020502215065062047,
|
|
"step": 4190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.6,
|
|
"completions/max_terminated_length": 275.6,
|
|
"completions/mean_length": 185.6,
|
|
"completions/mean_terminated_length": 185.6,
|
|
"completions/min_length": 114.3,
|
|
"completions/min_terminated_length": 114.3,
|
|
"entropy": 1.0363887690007687,
|
|
"epoch": 0.3861358830559897,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.004512786865234,
|
|
"kl": 0.048218690371140835,
|
|
"learning_rate": 4.91587688465711e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 18180288.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.48346473276615143,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.48346475064754485,
|
|
"sampling/importance_sampling_ratio/max": 1.270919132232666,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998312711715698,
|
|
"sampling/importance_sampling_ratio/min": 0.7961426794528961,
|
|
"sampling/sampling_logp_difference/max": 0.26231467723846436,
|
|
"sampling/sampling_logp_difference/mean": 0.02152935937047005,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"epoch": 0.3861358830559897,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0004363001745200698,
|
|
"eval_completions/max_length": 273.195462478185,
|
|
"eval_completions/max_terminated_length": 272.6561954624782,
|
|
"eval_completions/mean_length": 194.6014397905759,
|
|
"eval_completions/mean_terminated_length": 194.48382579011235,
|
|
"eval_completions/min_length": 125.17102966841186,
|
|
"eval_completions/min_terminated_length": 125.17102966841186,
|
|
"eval_entropy": 1.0418714970922387,
|
|
"eval_frac_reward_zero_std": 0.2844677137870855,
|
|
"eval_kl": 0.042036805220637435,
|
|
"eval_loss": 0.002092598704621196,
|
|
"eval_num_tokens": 18180288.0,
|
|
"eval_reward": 0.05606457242582897,
|
|
"eval_reward_std": 0.368489447584535,
|
|
"eval_rewards/reward_fn/mean": 0.05606457242582897,
|
|
"eval_rewards/reward_fn/std": 0.3684894585068521,
|
|
"eval_runtime": 1611.9064,
|
|
"eval_samples_per_second": 0.355,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2763111385375416,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999771187858848,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.77395340431007,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2815808745891012,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021152757574685477,
|
|
"eval_steps_per_second": 0.045,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 260.5,
|
|
"completions/max_terminated_length": 260.5,
|
|
"completions/mean_length": 197.9875,
|
|
"completions/mean_terminated_length": 197.9875,
|
|
"completions/min_length": 130.9,
|
|
"completions/min_terminated_length": 130.9,
|
|
"entropy": 1.003240905329585,
|
|
"epoch": 0.387055254206123,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.37614917755127,
|
|
"kl": 0.14689252984244378,
|
|
"learning_rate": 4.915223925150916e-06,
|
|
"loss": 0.0078,
|
|
"num_tokens": 18221527.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.29291952252388,
|
|
"rewards/reward_fn/mean": 0.2125,
|
|
"rewards/reward_fn/std": 0.2929195284843445,
|
|
"sampling/importance_sampling_ratio/max": 1.259115171432495,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999430298805236,
|
|
"sampling/importance_sampling_ratio/min": 0.7786946892738342,
|
|
"sampling/sampling_logp_difference/max": 0.26787192821502687,
|
|
"sampling/sampling_logp_difference/mean": 0.020318554993718863,
|
|
"step": 4210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 263.9,
|
|
"completions/max_terminated_length": 263.9,
|
|
"completions/mean_length": 186.3625,
|
|
"completions/mean_terminated_length": 186.3625,
|
|
"completions/min_length": 127.9,
|
|
"completions/min_terminated_length": 127.9,
|
|
"entropy": 1.0737589441239834,
|
|
"epoch": 0.3879746253562563,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.920482158660889,
|
|
"kl": 0.036211347358766945,
|
|
"learning_rate": 4.914568485016148e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 18261156.0,
|
|
"reward": -0.05625,
|
|
"reward_std": 0.40856347382068636,
|
|
"rewards/reward_fn/mean": -0.05625,
|
|
"rewards/reward_fn/std": 0.4085634797811508,
|
|
"sampling/importance_sampling_ratio/max": 1.2665090441703797,
|
|
"sampling/importance_sampling_ratio/mean": 1.000233346223831,
|
|
"sampling/importance_sampling_ratio/min": 0.7647046029567719,
|
|
"sampling/sampling_logp_difference/max": 0.2854976117610931,
|
|
"sampling/sampling_logp_difference/mean": 0.021909939125180244,
|
|
"step": 4220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.7,
|
|
"completions/max_terminated_length": 303.7,
|
|
"completions/mean_length": 215.5125,
|
|
"completions/mean_terminated_length": 215.5125,
|
|
"completions/min_length": 145.7,
|
|
"completions/min_terminated_length": 145.7,
|
|
"entropy": 1.0910020902752877,
|
|
"epoch": 0.3888939965063896,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.885628700256348,
|
|
"kl": 0.06614169023232534,
|
|
"learning_rate": 4.9139105649259945e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 18308045.0,
|
|
"reward": -0.3375,
|
|
"reward_std": 0.4683582723140717,
|
|
"rewards/reward_fn/mean": -0.3375,
|
|
"rewards/reward_fn/std": 0.46835830211639407,
|
|
"sampling/importance_sampling_ratio/max": 1.285937774181366,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004385709762573,
|
|
"sampling/importance_sampling_ratio/min": 0.8008832097053528,
|
|
"sampling/sampling_logp_difference/max": 0.26284568309783934,
|
|
"sampling/sampling_logp_difference/mean": 0.021850558184087276,
|
|
"step": 4230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 246.1,
|
|
"completions/max_terminated_length": 246.1,
|
|
"completions/mean_length": 180.125,
|
|
"completions/mean_terminated_length": 180.125,
|
|
"completions/min_length": 115.1,
|
|
"completions/min_terminated_length": 115.1,
|
|
"entropy": 1.0903586514294148,
|
|
"epoch": 0.3898133676565229,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1424350142478943,
|
|
"kl": 0.054549929895438255,
|
|
"learning_rate": 4.913250165556192e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 18347559.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.3483661413192749,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.3483661592006683,
|
|
"sampling/importance_sampling_ratio/max": 1.242164146900177,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999526917934418,
|
|
"sampling/importance_sampling_ratio/min": 0.7780108630657196,
|
|
"sampling/sampling_logp_difference/max": 0.2678714394569397,
|
|
"sampling/sampling_logp_difference/mean": 0.022136073932051657,
|
|
"step": 4240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.9,
|
|
"completions/max_terminated_length": 294.9,
|
|
"completions/mean_length": 200.1,
|
|
"completions/mean_terminated_length": 200.1,
|
|
"completions/min_length": 139.1,
|
|
"completions/min_terminated_length": 139.1,
|
|
"entropy": 1.0689876221120358,
|
|
"epoch": 0.39073273880665627,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.241642475128174,
|
|
"kl": 0.05478483536862768,
|
|
"learning_rate": 4.912587287585023e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 18389223.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.3257186770439148,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.32571869492530825,
|
|
"sampling/importance_sampling_ratio/max": 1.30518639087677,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006941497325896,
|
|
"sampling/importance_sampling_ratio/min": 0.7768790602684021,
|
|
"sampling/sampling_logp_difference/max": 0.2857847332954407,
|
|
"sampling/sampling_logp_difference/mean": 0.021296378038823605,
|
|
"step": 4250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.0,
|
|
"completions/max_terminated_length": 319.0,
|
|
"completions/mean_length": 226.6625,
|
|
"completions/mean_terminated_length": 226.6625,
|
|
"completions/min_length": 150.2,
|
|
"completions/min_terminated_length": 150.2,
|
|
"entropy": 1.116308617591858,
|
|
"epoch": 0.39165210995678956,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.940467357635498,
|
|
"kl": 0.052260009263409304,
|
|
"learning_rate": 4.911921931693317e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 18436252.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.40190111100673676,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.4019011229276657,
|
|
"sampling/importance_sampling_ratio/max": 1.2438296675682068,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005820214748382,
|
|
"sampling/importance_sampling_ratio/min": 0.7571478366851807,
|
|
"sampling/sampling_logp_difference/max": 0.2950997710227966,
|
|
"sampling/sampling_logp_difference/mean": 0.02172163166105747,
|
|
"step": 4260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 341.2,
|
|
"completions/max_terminated_length": 323.4,
|
|
"completions/mean_length": 220.85,
|
|
"completions/mean_terminated_length": 217.21964416503906,
|
|
"completions/min_length": 140.6,
|
|
"completions/min_terminated_length": 140.6,
|
|
"entropy": 1.1871419869363309,
|
|
"epoch": 0.39257148110692286,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.601526737213135,
|
|
"kl": 0.05504003544338047,
|
|
"learning_rate": 4.91125409856445e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 18480464.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.24116460382938384,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.24116460382938384,
|
|
"sampling/importance_sampling_ratio/max": 1.2678253769874572,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998971939086914,
|
|
"sampling/importance_sampling_ratio/min": 0.7594729900360108,
|
|
"sampling/sampling_logp_difference/max": 0.2952740788459778,
|
|
"sampling/sampling_logp_difference/mean": 0.022710669226944445,
|
|
"step": 4270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 320.3,
|
|
"completions/max_terminated_length": 320.3,
|
|
"completions/mean_length": 216.125,
|
|
"completions/mean_terminated_length": 216.125,
|
|
"completions/min_length": 140.2,
|
|
"completions/min_terminated_length": 140.2,
|
|
"entropy": 1.1805534809827805,
|
|
"epoch": 0.39349085225705616,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.15249940752983093,
|
|
"kl": 0.06016960216220468,
|
|
"learning_rate": 4.910583788884336e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 18523242.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.40020377933979034,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.40020379722118377,
|
|
"sampling/importance_sampling_ratio/max": 1.2664490342140198,
|
|
"sampling/importance_sampling_ratio/mean": 0.999782133102417,
|
|
"sampling/importance_sampling_ratio/min": 0.7839413523674011,
|
|
"sampling/sampling_logp_difference/max": 0.2572047233581543,
|
|
"sampling/sampling_logp_difference/mean": 0.0226281451061368,
|
|
"step": 4280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 310.9,
|
|
"completions/max_terminated_length": 290.8,
|
|
"completions/mean_length": 217.9625,
|
|
"completions/mean_terminated_length": 214.2500015258789,
|
|
"completions/min_length": 137.1,
|
|
"completions/min_terminated_length": 137.1,
|
|
"entropy": 1.0520618453621864,
|
|
"epoch": 0.3944102234071895,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.128655910491943,
|
|
"kl": 0.05966647712048143,
|
|
"learning_rate": 4.90991100334144e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 18567279.0,
|
|
"reward": -0.06875,
|
|
"reward_std": 0.392415389418602,
|
|
"rewards/reward_fn/mean": -0.06875,
|
|
"rewards/reward_fn/std": 0.3924154072999954,
|
|
"sampling/importance_sampling_ratio/max": 1.2285534381866454,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997376441955567,
|
|
"sampling/importance_sampling_ratio/min": 0.7765109121799469,
|
|
"sampling/sampling_logp_difference/max": 0.2538025975227356,
|
|
"sampling/sampling_logp_difference/mean": 0.02082740664482117,
|
|
"step": 4290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.9,
|
|
"completions/max_terminated_length": 305.9,
|
|
"completions/mean_length": 211.0,
|
|
"completions/mean_terminated_length": 211.0,
|
|
"completions/min_length": 143.3,
|
|
"completions/min_terminated_length": 143.3,
|
|
"entropy": 1.1813451118767262,
|
|
"epoch": 0.3953295945573228,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.12464196979999542,
|
|
"kl": 0.049402165017090736,
|
|
"learning_rate": 4.909235742626765e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 18613535.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.1334012657403946,
|
|
"sampling/importance_sampling_ratio/max": 1.313716423511505,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005593836307525,
|
|
"sampling/importance_sampling_ratio/min": 0.7994739353656769,
|
|
"sampling/sampling_logp_difference/max": 0.28026113510131834,
|
|
"sampling/sampling_logp_difference/mean": 0.021760441176593304,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.7,
|
|
"completions/max_terminated_length": 255.7,
|
|
"completions/mean_length": 195.325,
|
|
"completions/mean_terminated_length": 195.325,
|
|
"completions/min_length": 128.7,
|
|
"completions/min_terminated_length": 128.7,
|
|
"entropy": 1.1389208287000656,
|
|
"epoch": 0.3962489657074561,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.981790065765381,
|
|
"kl": 0.05124405643437058,
|
|
"learning_rate": 4.908558007433859e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 18656009.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.22853160798549652,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.228531613945961,
|
|
"sampling/importance_sampling_ratio/max": 1.2597496151924132,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997198283672333,
|
|
"sampling/importance_sampling_ratio/min": 0.760997986793518,
|
|
"sampling/sampling_logp_difference/max": 0.2984266757965088,
|
|
"sampling/sampling_logp_difference/mean": 0.02232794966548681,
|
|
"step": 4310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.4,
|
|
"completions/max_terminated_length": 294.4,
|
|
"completions/mean_length": 217.125,
|
|
"completions/mean_terminated_length": 217.125,
|
|
"completions/min_length": 148.1,
|
|
"completions/min_terminated_length": 148.1,
|
|
"entropy": 1.133064690977335,
|
|
"epoch": 0.3971683368575894,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1506035327911377,
|
|
"kl": 0.0594520561164245,
|
|
"learning_rate": 4.907877798458809e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 18701403.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.267578649520874,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.267578661441803,
|
|
"sampling/importance_sampling_ratio/max": 1.2770514369010926,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003732800483705,
|
|
"sampling/importance_sampling_ratio/min": 0.7623413741588593,
|
|
"sampling/sampling_logp_difference/max": 0.2804722785949707,
|
|
"sampling/sampling_logp_difference/mean": 0.022318172641098498,
|
|
"step": 4320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 279.5,
|
|
"completions/max_terminated_length": 277.5,
|
|
"completions/mean_length": 210.6125,
|
|
"completions/mean_terminated_length": 209.22678833007814,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 1.0104252938181162,
|
|
"epoch": 0.3980877080077227,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14306038618087769,
|
|
"kl": 0.03761104336590506,
|
|
"learning_rate": 4.907195116400245e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 18743748.0,
|
|
"reward": 0.26875,
|
|
"reward_std": 0.3223089873790741,
|
|
"rewards/reward_fn/mean": 0.26875,
|
|
"rewards/reward_fn/std": 0.32230899333953855,
|
|
"sampling/importance_sampling_ratio/max": 1.2912437438964843,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997095704078675,
|
|
"sampling/importance_sampling_ratio/min": 0.7927197873592376,
|
|
"sampling/sampling_logp_difference/max": 0.2589628338813782,
|
|
"sampling/sampling_logp_difference/mean": 0.02098599597811699,
|
|
"step": 4330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.4,
|
|
"completions/max_terminated_length": 309.4,
|
|
"completions/mean_length": 216.85,
|
|
"completions/mean_terminated_length": 216.85,
|
|
"completions/min_length": 146.5,
|
|
"completions/min_terminated_length": 146.5,
|
|
"entropy": 1.0362945929169656,
|
|
"epoch": 0.39900707915785605,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.333477020263672,
|
|
"kl": 0.04038164290832356,
|
|
"learning_rate": 4.906509961959337e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 18788640.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.3558045417070389,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.35580454766750336,
|
|
"sampling/importance_sampling_ratio/max": 1.2765664935112,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996415257453919,
|
|
"sampling/importance_sampling_ratio/min": 0.7695736050605774,
|
|
"sampling/sampling_logp_difference/max": 0.2857661724090576,
|
|
"sampling/sampling_logp_difference/mean": 0.02036453168839216,
|
|
"step": 4340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.6,
|
|
"completions/max_terminated_length": 275.6,
|
|
"completions/mean_length": 195.4375,
|
|
"completions/mean_terminated_length": 195.4375,
|
|
"completions/min_length": 130.8,
|
|
"completions/min_terminated_length": 130.8,
|
|
"entropy": 1.1462503150105476,
|
|
"epoch": 0.39992645030798935,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.944637298583984,
|
|
"kl": 0.05211303613614291,
|
|
"learning_rate": 4.9058223358397924e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 18831995.0,
|
|
"reward": -0.275,
|
|
"reward_std": 0.30157616436481477,
|
|
"rewards/reward_fn/mean": -0.275,
|
|
"rewards/reward_fn/std": 0.3015761703252792,
|
|
"sampling/importance_sampling_ratio/max": 1.2654507160186768,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993409037590026,
|
|
"sampling/importance_sampling_ratio/min": 0.786357718706131,
|
|
"sampling/sampling_logp_difference/max": 0.2626077175140381,
|
|
"sampling/sampling_logp_difference/mean": 0.022217776626348495,
|
|
"step": 4350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.3,
|
|
"completions/max_terminated_length": 281.3,
|
|
"completions/mean_length": 209.225,
|
|
"completions/mean_terminated_length": 209.225,
|
|
"completions/min_length": 132.4,
|
|
"completions/min_terminated_length": 132.4,
|
|
"entropy": 1.1078680358827113,
|
|
"epoch": 0.40084582145812264,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.837707996368408,
|
|
"kl": 0.04209608676610514,
|
|
"learning_rate": 4.905132238747858e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 18874613.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.4246234506368637,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.42462346851825716,
|
|
"sampling/importance_sampling_ratio/max": 1.2636314630508423,
|
|
"sampling/importance_sampling_ratio/mean": 1.000173419713974,
|
|
"sampling/importance_sampling_ratio/min": 0.8058146178722382,
|
|
"sampling/sampling_logp_difference/max": 0.24692773818969727,
|
|
"sampling/sampling_logp_difference/mean": 0.021321115642786027,
|
|
"step": 4360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 254.8,
|
|
"completions/max_terminated_length": 254.8,
|
|
"completions/mean_length": 181.525,
|
|
"completions/mean_terminated_length": 181.525,
|
|
"completions/min_length": 126.4,
|
|
"completions/min_terminated_length": 126.4,
|
|
"entropy": 1.0825784616172314,
|
|
"epoch": 0.40176519260825594,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.313657283782959,
|
|
"kl": 0.040845442190766336,
|
|
"learning_rate": 4.904439671392321e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 18915055.0,
|
|
"reward": -0.05625,
|
|
"reward_std": 0.40697675943374634,
|
|
"rewards/reward_fn/mean": -0.05625,
|
|
"rewards/reward_fn/std": 0.40697677731513976,
|
|
"sampling/importance_sampling_ratio/max": 1.2808258652687072,
|
|
"sampling/importance_sampling_ratio/mean": 0.999857610464096,
|
|
"sampling/importance_sampling_ratio/min": 0.756984430551529,
|
|
"sampling/sampling_logp_difference/max": 0.3004852533340454,
|
|
"sampling/sampling_logp_difference/mean": 0.021898503974080087,
|
|
"step": 4370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.0,
|
|
"completions/max_terminated_length": 279.0,
|
|
"completions/mean_length": 193.8625,
|
|
"completions/mean_terminated_length": 193.8625,
|
|
"completions/min_length": 111.6,
|
|
"completions/min_terminated_length": 111.6,
|
|
"entropy": 1.1068538449704648,
|
|
"epoch": 0.40268456375838924,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.112775519490242,
|
|
"kl": 0.05176783774513751,
|
|
"learning_rate": 4.9037446344845e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 18958972.0,
|
|
"reward": 0.00625,
|
|
"reward_std": 0.35268451273441315,
|
|
"rewards/reward_fn/mean": 0.00625,
|
|
"rewards/reward_fn/std": 0.35268453061580657,
|
|
"sampling/importance_sampling_ratio/max": 1.2986793398857117,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998490869998932,
|
|
"sampling/importance_sampling_ratio/min": 0.7972592234611511,
|
|
"sampling/sampling_logp_difference/max": 0.2788501262664795,
|
|
"sampling/sampling_logp_difference/mean": 0.021204747818410398,
|
|
"step": 4380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 322.5,
|
|
"completions/max_terminated_length": 322.5,
|
|
"completions/mean_length": 232.2625,
|
|
"completions/mean_terminated_length": 232.2625,
|
|
"completions/min_length": 147.3,
|
|
"completions/min_terminated_length": 147.3,
|
|
"entropy": 1.1205988615751266,
|
|
"epoch": 0.4036039349085226,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.673587322235107,
|
|
"kl": 0.053909664938692,
|
|
"learning_rate": 4.903047128738257e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 19006401.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.5421775996685028,
|
|
"rewards/reward_fn/mean": 0.03125,
|
|
"rewards/reward_fn/std": 0.5421776235103607,
|
|
"sampling/importance_sampling_ratio/max": 1.2923802137374878,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001834630966187,
|
|
"sampling/importance_sampling_ratio/min": 0.7839033603668213,
|
|
"sampling/sampling_logp_difference/max": 0.2786647915840149,
|
|
"sampling/sampling_logp_difference/mean": 0.021829541958868503,
|
|
"step": 4390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.5,
|
|
"completions/max_terminated_length": 319.5,
|
|
"completions/mean_length": 228.5125,
|
|
"completions/mean_terminated_length": 228.5125,
|
|
"completions/min_length": 156.6,
|
|
"completions/min_terminated_length": 156.6,
|
|
"entropy": 1.1149950563907622,
|
|
"epoch": 0.4045233060586559,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.860590934753418,
|
|
"kl": 0.044792854588013145,
|
|
"learning_rate": 4.902347154869984e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 19051914.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.4301289528608322,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.43012896478176116,
|
|
"sampling/importance_sampling_ratio/max": 1.2460772275924683,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999591708183289,
|
|
"sampling/importance_sampling_ratio/min": 0.7567124605178833,
|
|
"sampling/sampling_logp_difference/max": 0.2889558672904968,
|
|
"sampling/sampling_logp_difference/mean": 0.022890967689454554,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"epoch": 0.4045233060586559,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0017452006980802793,
|
|
"eval_completions/max_length": 300.35602094240835,
|
|
"eval_completions/max_terminated_length": 298.92844677137873,
|
|
"eval_completions/mean_length": 210.85296684118674,
|
|
"eval_completions/mean_terminated_length": 210.42673288714823,
|
|
"eval_completions/min_length": 135.087260034904,
|
|
"eval_completions/min_terminated_length": 135.087260034904,
|
|
"eval_entropy": 1.0885618127765455,
|
|
"eval_frac_reward_zero_std": 0.25654450261780104,
|
|
"eval_kl": 0.043799373920987414,
|
|
"eval_loss": 0.002299663843587041,
|
|
"eval_num_tokens": 19051914.0,
|
|
"eval_reward": 0.05399214659685864,
|
|
"eval_reward_std": 0.3752189955786261,
|
|
"eval_rewards/reward_fn/mean": 0.05399214659685864,
|
|
"eval_rewards/reward_fn/std": 0.3752190072290976,
|
|
"eval_runtime": 1751.794,
|
|
"eval_samples_per_second": 0.327,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2712574837303494,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000437070561954,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7767157262324454,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2802350786343919,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02125193019369926,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 267.0,
|
|
"completions/max_terminated_length": 267.0,
|
|
"completions/mean_length": 190.9375,
|
|
"completions/mean_terminated_length": 190.9375,
|
|
"completions/min_length": 124.1,
|
|
"completions/min_terminated_length": 124.1,
|
|
"entropy": 1.0574560962617396,
|
|
"epoch": 0.4054426772087892,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.079658508300781,
|
|
"kl": 0.05369033552706241,
|
|
"learning_rate": 4.901644713598612e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 19090869.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.29291952252388,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.2929195284843445,
|
|
"sampling/importance_sampling_ratio/max": 1.2977915167808534,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996715247631073,
|
|
"sampling/importance_sampling_ratio/min": 0.770642626285553,
|
|
"sampling/sampling_logp_difference/max": 0.2975643157958984,
|
|
"sampling/sampling_logp_difference/mean": 0.022062713094055653,
|
|
"step": 4410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.7,
|
|
"completions/max_terminated_length": 296.7,
|
|
"completions/mean_length": 207.025,
|
|
"completions/mean_terminated_length": 207.025,
|
|
"completions/min_length": 128.6,
|
|
"completions/min_terminated_length": 128.6,
|
|
"entropy": 0.9994460038840771,
|
|
"epoch": 0.4063620483589225,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.009789943695068,
|
|
"kl": 0.03408454702002928,
|
|
"learning_rate": 4.9009398056456045e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 19132407.0,
|
|
"reward": -0.2,
|
|
"reward_std": 0.6041975826025009,
|
|
"rewards/reward_fn/mean": -0.2,
|
|
"rewards/reward_fn/std": 0.6041975945234299,
|
|
"sampling/importance_sampling_ratio/max": 1.331460964679718,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997465968132019,
|
|
"sampling/importance_sampling_ratio/min": 0.7817839324474335,
|
|
"sampling/sampling_logp_difference/max": 0.3036469578742981,
|
|
"sampling/sampling_logp_difference/mean": 0.022120250202715398,
|
|
"step": 4420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 331.0,
|
|
"completions/max_terminated_length": 323.2,
|
|
"completions/mean_length": 209.5875,
|
|
"completions/mean_terminated_length": 206.29107208251952,
|
|
"completions/min_length": 139.1,
|
|
"completions/min_terminated_length": 139.1,
|
|
"entropy": 1.1555320933461188,
|
|
"epoch": 0.40728141950905583,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.998329162597656,
|
|
"kl": 0.04895743831875734,
|
|
"learning_rate": 4.900232431734957e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 19176246.0,
|
|
"reward": -0.00625,
|
|
"reward_std": 0.5835410565137863,
|
|
"rewards/reward_fn/mean": -0.00625,
|
|
"rewards/reward_fn/std": 0.5835410743951798,
|
|
"sampling/importance_sampling_ratio/max": 1.24947350025177,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002531945705413,
|
|
"sampling/importance_sampling_ratio/min": 0.7625175654888153,
|
|
"sampling/sampling_logp_difference/max": 0.2922655284404755,
|
|
"sampling/sampling_logp_difference/mean": 0.02250595297664404,
|
|
"step": 4430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.9,
|
|
"completions/max_terminated_length": 311.9,
|
|
"completions/mean_length": 224.5625,
|
|
"completions/mean_terminated_length": 224.5625,
|
|
"completions/min_length": 131.2,
|
|
"completions/min_terminated_length": 131.2,
|
|
"entropy": 1.0103722549974918,
|
|
"epoch": 0.4082007906591891,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.733419895172119,
|
|
"kl": 0.040536357468226925,
|
|
"learning_rate": 4.899522592593199e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 19218419.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.4372564494609833,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.4372564613819122,
|
|
"sampling/importance_sampling_ratio/max": 1.283979845046997,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002211093902589,
|
|
"sampling/importance_sampling_ratio/min": 0.7649264752864837,
|
|
"sampling/sampling_logp_difference/max": 0.30294662714004517,
|
|
"sampling/sampling_logp_difference/mean": 0.020980802923440935,
|
|
"step": 4440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 329.4,
|
|
"completions/max_terminated_length": 329.4,
|
|
"completions/mean_length": 224.725,
|
|
"completions/mean_terminated_length": 224.725,
|
|
"completions/min_length": 153.6,
|
|
"completions/min_terminated_length": 153.6,
|
|
"entropy": 1.1161303229629993,
|
|
"epoch": 0.4091201618093224,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.516714096069336,
|
|
"kl": 0.060422635986469686,
|
|
"learning_rate": 4.898810288949394e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 19263629.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.2635735034942627,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.2635735034942627,
|
|
"sampling/importance_sampling_ratio/max": 1.2781274795532227,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999723732471466,
|
|
"sampling/importance_sampling_ratio/min": 0.7407774329185486,
|
|
"sampling/sampling_logp_difference/max": 0.33312050104141233,
|
|
"sampling/sampling_logp_difference/mean": 0.022814949974417686,
|
|
"step": 4450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 317.7,
|
|
"completions/max_terminated_length": 317.7,
|
|
"completions/mean_length": 214.5375,
|
|
"completions/mean_terminated_length": 214.5375,
|
|
"completions/min_length": 142.9,
|
|
"completions/min_terminated_length": 142.9,
|
|
"entropy": 1.046873278170824,
|
|
"epoch": 0.4100395329594557,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.215446472167969,
|
|
"kl": 0.047878540342207995,
|
|
"learning_rate": 4.898095521535134e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 19309192.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.4269921988248825,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.42699221670627596,
|
|
"sampling/importance_sampling_ratio/max": 1.2364967584609985,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997314095497132,
|
|
"sampling/importance_sampling_ratio/min": 0.783577561378479,
|
|
"sampling/sampling_logp_difference/max": 0.24711763858795166,
|
|
"sampling/sampling_logp_difference/mean": 0.020391737669706346,
|
|
"step": 4460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.7,
|
|
"completions/max_terminated_length": 311.7,
|
|
"completions/mean_length": 244.8875,
|
|
"completions/mean_terminated_length": 244.8875,
|
|
"completions/min_length": 177.6,
|
|
"completions/min_terminated_length": 177.6,
|
|
"entropy": 1.1675960309803486,
|
|
"epoch": 0.410958904109589,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.997447490692139,
|
|
"kl": 0.042925568885402755,
|
|
"learning_rate": 4.897378291084543e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 19359831.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.26472542583942416,
|
|
"rewards/reward_fn/mean": 0.35,
|
|
"rewards/reward_fn/std": 0.2647254317998886,
|
|
"sampling/importance_sampling_ratio/max": 1.2740905284881592,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000465214252472,
|
|
"sampling/importance_sampling_ratio/min": 0.7818454623222351,
|
|
"sampling/sampling_logp_difference/max": 0.2711315453052521,
|
|
"sampling/sampling_logp_difference/mean": 0.022124696336686612,
|
|
"step": 4470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 329.1,
|
|
"completions/max_terminated_length": 329.1,
|
|
"completions/mean_length": 240.7125,
|
|
"completions/mean_terminated_length": 240.7125,
|
|
"completions/min_length": 151.6,
|
|
"completions/min_terminated_length": 151.6,
|
|
"entropy": 1.146951925754547,
|
|
"epoch": 0.41187827525972237,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.70084285736084,
|
|
"kl": 0.04147570205386728,
|
|
"learning_rate": 4.896658598334273e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 19408376.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.45269069373607634,
|
|
"rewards/reward_fn/mean": 0.2875,
|
|
"rewards/reward_fn/std": 0.45269071161746977,
|
|
"sampling/importance_sampling_ratio/max": 1.3358041524887085,
|
|
"sampling/importance_sampling_ratio/mean": 1.00005641579628,
|
|
"sampling/importance_sampling_ratio/min": 0.7707864463329315,
|
|
"sampling/sampling_logp_difference/max": 0.31489416360855105,
|
|
"sampling/sampling_logp_difference/mean": 0.022473588399589062,
|
|
"step": 4480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.6,
|
|
"completions/max_terminated_length": 291.6,
|
|
"completions/mean_length": 217.875,
|
|
"completions/mean_terminated_length": 217.875,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 1.053855038434267,
|
|
"epoch": 0.41279764640985567,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 7.070992946624756,
|
|
"kl": 0.03861591417808086,
|
|
"learning_rate": 4.8959364440235065e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 19451510.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.22853160798549652,
|
|
"rewards/reward_fn/mean": 0.45,
|
|
"rewards/reward_fn/std": 0.228531613945961,
|
|
"sampling/importance_sampling_ratio/max": 1.2772159934043885,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004786193370818,
|
|
"sampling/importance_sampling_ratio/min": 0.7876056969165802,
|
|
"sampling/sampling_logp_difference/max": 0.2735481858253479,
|
|
"sampling/sampling_logp_difference/mean": 0.021447963453829287,
|
|
"step": 4490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.9,
|
|
"completions/max_terminated_length": 275.9,
|
|
"completions/mean_length": 186.65,
|
|
"completions/mean_terminated_length": 186.65,
|
|
"completions/min_length": 118.3,
|
|
"completions/min_terminated_length": 118.3,
|
|
"entropy": 1.1102378405630589,
|
|
"epoch": 0.41371701755998896,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.12575484812259674,
|
|
"kl": 0.0463074947358109,
|
|
"learning_rate": 4.8952118288939555e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 19494154.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.3998240172863007,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.39982402324676514,
|
|
"sampling/importance_sampling_ratio/max": 1.259126365184784,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999365329742431,
|
|
"sampling/importance_sampling_ratio/min": 0.7282272160053254,
|
|
"sampling/sampling_logp_difference/max": 0.32773895263671876,
|
|
"sampling/sampling_logp_difference/mean": 0.022174909338355066,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.5,
|
|
"completions/max_terminated_length": 307.5,
|
|
"completions/mean_length": 211.2875,
|
|
"completions/mean_terminated_length": 211.2875,
|
|
"completions/min_length": 137.7,
|
|
"completions/min_terminated_length": 137.7,
|
|
"entropy": 1.043286493420601,
|
|
"epoch": 0.41463638871012226,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.417031764984131,
|
|
"kl": 0.030020576249808074,
|
|
"learning_rate": 4.894484753689855e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 19535025.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.4276267558336258,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.4276267737150192,
|
|
"sampling/importance_sampling_ratio/max": 1.2816107273101807,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998485207557678,
|
|
"sampling/importance_sampling_ratio/min": 0.726537162065506,
|
|
"sampling/sampling_logp_difference/max": 0.34288318157196046,
|
|
"sampling/sampling_logp_difference/mean": 0.021475453302264212,
|
|
"step": 4510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.9,
|
|
"completions/max_terminated_length": 273.9,
|
|
"completions/mean_length": 206.7,
|
|
"completions/mean_terminated_length": 206.7,
|
|
"completions/min_length": 143.8,
|
|
"completions/min_terminated_length": 143.8,
|
|
"entropy": 1.091755896061659,
|
|
"epoch": 0.41555575986025556,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.505802154541016,
|
|
"kl": 0.0328896620485466,
|
|
"learning_rate": 4.8937552191579716e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 19578841.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.47562678158283234,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.4756267935037613,
|
|
"sampling/importance_sampling_ratio/max": 1.2823106527328492,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006014227867126,
|
|
"sampling/importance_sampling_ratio/min": 0.7985706090927124,
|
|
"sampling/sampling_logp_difference/max": 0.2697652578353882,
|
|
"sampling/sampling_logp_difference/mean": 0.021643362194299697,
|
|
"step": 4520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.6,
|
|
"completions/max_terminated_length": 326.6,
|
|
"completions/mean_length": 213.3125,
|
|
"completions/mean_terminated_length": 213.3125,
|
|
"completions/min_length": 129.8,
|
|
"completions/min_terminated_length": 129.8,
|
|
"entropy": 1.0473481640219688,
|
|
"epoch": 0.4164751310103889,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.6811933517456055,
|
|
"kl": 0.05285923631163314,
|
|
"learning_rate": 4.893023226047595e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 19624122.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4038677543401718,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.4038677603006363,
|
|
"sampling/importance_sampling_ratio/max": 1.2622378826141358,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996782839298248,
|
|
"sampling/importance_sampling_ratio/min": 0.7822768211364746,
|
|
"sampling/sampling_logp_difference/max": 0.25872786045074464,
|
|
"sampling/sampling_logp_difference/mean": 0.02213543336838484,
|
|
"step": 4530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 263.4,
|
|
"completions/max_terminated_length": 263.4,
|
|
"completions/mean_length": 189.4875,
|
|
"completions/mean_terminated_length": 189.4875,
|
|
"completions/min_length": 127.6,
|
|
"completions/min_terminated_length": 127.6,
|
|
"entropy": 0.9849121943116188,
|
|
"epoch": 0.4173945021605222,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.543390274047852,
|
|
"kl": 0.036978136870311575,
|
|
"learning_rate": 4.892288775110542e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 19663361.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.4437026858329773,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.4437026917934418,
|
|
"sampling/importance_sampling_ratio/max": 1.261613941192627,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005469858646392,
|
|
"sampling/importance_sampling_ratio/min": 0.7773848116397858,
|
|
"sampling/sampling_logp_difference/max": 0.2585180938243866,
|
|
"sampling/sampling_logp_difference/mean": 0.021519438363611698,
|
|
"step": 4540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.9,
|
|
"completions/max_terminated_length": 299.9,
|
|
"completions/mean_length": 208.225,
|
|
"completions/mean_terminated_length": 208.225,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 1.0951564125716686,
|
|
"epoch": 0.4183138733106555,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1212257593870163,
|
|
"kl": 0.03991349912248552,
|
|
"learning_rate": 4.891551867101151e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 19707235.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.3504155069589615,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.35041551291942596,
|
|
"sampling/importance_sampling_ratio/max": 1.2868492960929871,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000524044036865,
|
|
"sampling/importance_sampling_ratio/min": 0.7804197371006012,
|
|
"sampling/sampling_logp_difference/max": 0.2710308790206909,
|
|
"sampling/sampling_logp_difference/mean": 0.023127567395567893,
|
|
"step": 4550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.3,
|
|
"completions/max_terminated_length": 307.3,
|
|
"completions/mean_length": 203.1125,
|
|
"completions/mean_terminated_length": 203.1125,
|
|
"completions/min_length": 138.1,
|
|
"completions/min_terminated_length": 138.1,
|
|
"entropy": 1.132548500597477,
|
|
"epoch": 0.4192332444607888,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.4325469732284546,
|
|
"kl": 0.04296427817316726,
|
|
"learning_rate": 4.890812502776285e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 19749236.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.32111361920833587,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.3211136251688004,
|
|
"sampling/importance_sampling_ratio/max": 1.2543392539024354,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999577879905701,
|
|
"sampling/importance_sampling_ratio/min": 0.7559376358985901,
|
|
"sampling/sampling_logp_difference/max": 0.3011161208152771,
|
|
"sampling/sampling_logp_difference/mean": 0.022630528546869755,
|
|
"step": 4560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 362.5,
|
|
"completions/max_terminated_length": 362.5,
|
|
"completions/mean_length": 242.5375,
|
|
"completions/mean_terminated_length": 242.5375,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 1.0516948409378528,
|
|
"epoch": 0.42015261561092215,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.431897163391113,
|
|
"kl": 0.044447302498156205,
|
|
"learning_rate": 4.8900706828953335e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 19799527.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.3011962354183197,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.3011962354183197,
|
|
"sampling/importance_sampling_ratio/max": 1.2429888486862182,
|
|
"sampling/importance_sampling_ratio/mean": 0.9991742849349976,
|
|
"sampling/importance_sampling_ratio/min": 0.7532529413700104,
|
|
"sampling/sampling_logp_difference/max": 0.2950943112373352,
|
|
"sampling/sampling_logp_difference/mean": 0.02096956018358469,
|
|
"step": 4570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.8,
|
|
"completions/max_terminated_length": 300.8,
|
|
"completions/mean_length": 214.2375,
|
|
"completions/mean_terminated_length": 214.2375,
|
|
"completions/min_length": 130.6,
|
|
"completions/min_terminated_length": 130.6,
|
|
"entropy": 1.1176028296351432,
|
|
"epoch": 0.42107198676105545,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.951211452484131,
|
|
"kl": 0.044005175190977755,
|
|
"learning_rate": 4.8893264082202015e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 19843434.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.3501461952924728,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.35014620125293733,
|
|
"sampling/importance_sampling_ratio/max": 1.2603368520736695,
|
|
"sampling/importance_sampling_ratio/mean": 1.000041949748993,
|
|
"sampling/importance_sampling_ratio/min": 0.7734138786792755,
|
|
"sampling/sampling_logp_difference/max": 0.2829380989074707,
|
|
"sampling/sampling_logp_difference/mean": 0.02187867760658264,
|
|
"step": 4580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.5,
|
|
"completions/max_terminated_length": 297.5,
|
|
"completions/mean_length": 199.1,
|
|
"completions/mean_terminated_length": 199.1,
|
|
"completions/min_length": 136.4,
|
|
"completions/min_terminated_length": 136.4,
|
|
"entropy": 1.0559039577841758,
|
|
"epoch": 0.42199135791118875,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 8.331087112426758,
|
|
"kl": 0.05335252875229344,
|
|
"learning_rate": 4.888579679515321e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 19886554.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.4403586119413376,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.4403586357831955,
|
|
"sampling/importance_sampling_ratio/max": 1.3077358245849608,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999307096004486,
|
|
"sampling/importance_sampling_ratio/min": 0.7760605692863465,
|
|
"sampling/sampling_logp_difference/max": 0.2890470206737518,
|
|
"sampling/sampling_logp_difference/mean": 0.022006846405565737,
|
|
"step": 4590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 338.7,
|
|
"completions/max_terminated_length": 338.7,
|
|
"completions/mean_length": 218.2,
|
|
"completions/mean_terminated_length": 218.2,
|
|
"completions/min_length": 124.4,
|
|
"completions/min_terminated_length": 124.4,
|
|
"entropy": 1.1244510293006897,
|
|
"epoch": 0.42291072906132204,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.269723415374756,
|
|
"kl": 0.05518236457137391,
|
|
"learning_rate": 4.887830497547641e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 19932002.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.45299161076545713,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4529916286468506,
|
|
"sampling/importance_sampling_ratio/max": 1.3063086986541748,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997603058815002,
|
|
"sampling/importance_sampling_ratio/min": 0.717989158630371,
|
|
"sampling/sampling_logp_difference/max": 0.364738917350769,
|
|
"sampling/sampling_logp_difference/mean": 0.02262552436441183,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"epoch": 0.42291072906132204,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002617801047120419,
|
|
"eval_completions/max_length": 311.68411867364745,
|
|
"eval_completions/max_terminated_length": 310.0,
|
|
"eval_completions/mean_length": 220.85732984293193,
|
|
"eval_completions/mean_terminated_length": 220.26623053159597,
|
|
"eval_completions/min_length": 142.27399650959862,
|
|
"eval_completions/min_terminated_length": 142.27399650959862,
|
|
"eval_entropy": 1.1087293722138563,
|
|
"eval_frac_reward_zero_std": 0.27399650959860383,
|
|
"eval_kl": 0.044565417282088265,
|
|
"eval_loss": 0.002313245553523302,
|
|
"eval_num_tokens": 19932002.0,
|
|
"eval_reward": 0.05355584642233857,
|
|
"eval_reward_std": 0.35679650665577795,
|
|
"eval_rewards/reward_fn/mean": 0.05355584642233857,
|
|
"eval_rewards/reward_fn/std": 0.35679651518558747,
|
|
"eval_runtime": 1807.2424,
|
|
"eval_samples_per_second": 0.317,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2738187360929984,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000183957825572,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7690312488436907,
|
|
"eval_sampling/sampling_logp_difference/max": 0.28902619817078007,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02185382135374956,
|
|
"eval_steps_per_second": 0.04,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.1,
|
|
"completions/max_terminated_length": 289.1,
|
|
"completions/mean_length": 212.475,
|
|
"completions/mean_terminated_length": 212.475,
|
|
"completions/min_length": 153.4,
|
|
"completions/min_terminated_length": 153.4,
|
|
"entropy": 1.109876587241888,
|
|
"epoch": 0.42383010021145534,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.14331884682178497,
|
|
"kl": 0.04352916833013296,
|
|
"learning_rate": 4.887078863086633e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 19974944.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.43530933260917665,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.43530936241149903,
|
|
"sampling/importance_sampling_ratio/max": 1.276979398727417,
|
|
"sampling/importance_sampling_ratio/mean": 0.999796575307846,
|
|
"sampling/importance_sampling_ratio/min": 0.7827986299991607,
|
|
"sampling/sampling_logp_difference/max": 0.27153608202934265,
|
|
"sampling/sampling_logp_difference/mean": 0.022779017314314844,
|
|
"step": 4610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 347.5,
|
|
"completions/max_terminated_length": 347.5,
|
|
"completions/mean_length": 239.2375,
|
|
"completions/mean_terminated_length": 239.2375,
|
|
"completions/min_length": 166.6,
|
|
"completions/min_terminated_length": 166.6,
|
|
"entropy": 1.1225914478302002,
|
|
"epoch": 0.4247494713615887,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.802147388458252,
|
|
"kl": 0.0531507924082689,
|
|
"learning_rate": 4.886324776904284e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 20021827.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.19317626953125,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.19317627549171448,
|
|
"sampling/importance_sampling_ratio/max": 1.3378105282783508,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999402284622192,
|
|
"sampling/importance_sampling_ratio/min": 0.7696754157543182,
|
|
"sampling/sampling_logp_difference/max": 0.33963828086853026,
|
|
"sampling/sampling_logp_difference/mean": 0.022954441793262958,
|
|
"step": 4620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 335.1,
|
|
"completions/max_terminated_length": 335.1,
|
|
"completions/mean_length": 228.2125,
|
|
"completions/mean_terminated_length": 228.2125,
|
|
"completions/min_length": 139.9,
|
|
"completions/min_terminated_length": 139.9,
|
|
"entropy": 1.069108320772648,
|
|
"epoch": 0.425668842511722,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.100437164306641,
|
|
"kl": 0.05007398951565847,
|
|
"learning_rate": 4.885568239775101e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 20066420.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.45408297181129453,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.4540829837322235,
|
|
"sampling/importance_sampling_ratio/max": 1.24729722738266,
|
|
"sampling/importance_sampling_ratio/mean": 0.999903815984726,
|
|
"sampling/importance_sampling_ratio/min": 0.7635926723480224,
|
|
"sampling/sampling_logp_difference/max": 0.2841993570327759,
|
|
"sampling/sampling_logp_difference/mean": 0.021710199117660523,
|
|
"step": 4630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.1,
|
|
"completions/max_terminated_length": 315.1,
|
|
"completions/mean_length": 225.4,
|
|
"completions/mean_terminated_length": 225.4,
|
|
"completions/min_length": 133.0,
|
|
"completions/min_terminated_length": 133.0,
|
|
"entropy": 1.1081184215843678,
|
|
"epoch": 0.4265882136618553,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.13854734599590302,
|
|
"kl": 0.04665623733308166,
|
|
"learning_rate": 4.884809252476111e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 20111724.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.43781724870204924,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.4378172606229782,
|
|
"sampling/importance_sampling_ratio/max": 1.274485993385315,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005553781986236,
|
|
"sampling/importance_sampling_ratio/min": 0.7626090228557587,
|
|
"sampling/sampling_logp_difference/max": 0.29525926113128664,
|
|
"sampling/sampling_logp_difference/mean": 0.022124290838837622,
|
|
"step": 4640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.6,
|
|
"completions/max_terminated_length": 306.6,
|
|
"completions/mean_length": 217.4125,
|
|
"completions/mean_terminated_length": 217.4125,
|
|
"completions/min_length": 122.3,
|
|
"completions/min_terminated_length": 122.3,
|
|
"entropy": 1.1606886588037013,
|
|
"epoch": 0.4275075848119886,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.082587718963623,
|
|
"kl": 0.05521926496876404,
|
|
"learning_rate": 4.884047815786852e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 20159229.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.31461594700813295,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.3146159708499908,
|
|
"sampling/importance_sampling_ratio/max": 1.325911819934845,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003619074821473,
|
|
"sampling/importance_sampling_ratio/min": 0.7949834585189819,
|
|
"sampling/sampling_logp_difference/max": 0.28434593677520753,
|
|
"sampling/sampling_logp_difference/mean": 0.02268270291388035,
|
|
"step": 4650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.5,
|
|
"completions/max_terminated_length": 315.5,
|
|
"completions/mean_length": 222.6375,
|
|
"completions/mean_terminated_length": 222.6375,
|
|
"completions/min_length": 153.4,
|
|
"completions/min_terminated_length": 153.4,
|
|
"entropy": 1.1878398574888707,
|
|
"epoch": 0.42842695596212194,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.096914768218994,
|
|
"kl": 0.05556288667721674,
|
|
"learning_rate": 4.883283930489381e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 20204544.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.3554430425167084,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.3554430603981018,
|
|
"sampling/importance_sampling_ratio/max": 1.229722809791565,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001407861709595,
|
|
"sampling/importance_sampling_ratio/min": 0.7895393967628479,
|
|
"sampling/sampling_logp_difference/max": 0.2393401861190796,
|
|
"sampling/sampling_logp_difference/mean": 0.022674289904534815,
|
|
"step": 4660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 310.5,
|
|
"completions/max_terminated_length": 303.2,
|
|
"completions/mean_length": 216.725,
|
|
"completions/mean_terminated_length": 213.97321472167968,
|
|
"completions/min_length": 135.8,
|
|
"completions/min_terminated_length": 135.8,
|
|
"entropy": 1.0784543931484223,
|
|
"epoch": 0.42934632711225523,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13704660534858704,
|
|
"kl": 0.06340418923064135,
|
|
"learning_rate": 4.88251759736827e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 20250114.0,
|
|
"reward": 0.11875,
|
|
"reward_std": 0.27900843024253846,
|
|
"rewards/reward_fn/mean": 0.11875,
|
|
"rewards/reward_fn/std": 0.2790084481239319,
|
|
"sampling/importance_sampling_ratio/max": 1.3357831954956054,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006938338279725,
|
|
"sampling/importance_sampling_ratio/min": 0.7587795197963715,
|
|
"sampling/sampling_logp_difference/max": 0.3546781659126282,
|
|
"sampling/sampling_logp_difference/mean": 0.02190141975879669,
|
|
"step": 4670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 318.3,
|
|
"completions/max_terminated_length": 318.3,
|
|
"completions/mean_length": 220.8875,
|
|
"completions/mean_terminated_length": 220.8875,
|
|
"completions/min_length": 134.5,
|
|
"completions/min_terminated_length": 134.5,
|
|
"entropy": 1.0997953221201897,
|
|
"epoch": 0.43026569826238853,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.3704142570495605,
|
|
"kl": 0.06075691705918871,
|
|
"learning_rate": 4.881748817210604e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 20296049.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.26849985122680664,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.2684998571872711,
|
|
"sampling/importance_sampling_ratio/max": 1.2410308361053466,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001990437507629,
|
|
"sampling/importance_sampling_ratio/min": 0.7861220121383667,
|
|
"sampling/sampling_logp_difference/max": 0.24477518796920777,
|
|
"sampling/sampling_logp_difference/mean": 0.02236610483378172,
|
|
"step": 4680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.6,
|
|
"completions/max_terminated_length": 305.6,
|
|
"completions/mean_length": 215.2375,
|
|
"completions/mean_terminated_length": 215.2375,
|
|
"completions/min_length": 138.0,
|
|
"completions/min_terminated_length": 138.0,
|
|
"entropy": 1.0569058526307344,
|
|
"epoch": 0.4311850694125218,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.1456536501646042,
|
|
"kl": 0.04264894136576913,
|
|
"learning_rate": 4.880977590805983e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 20340132.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.20411193668842315,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.20411194264888763,
|
|
"sampling/importance_sampling_ratio/max": 1.3091317653656005,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999290585517884,
|
|
"sampling/importance_sampling_ratio/min": 0.7606586873531341,
|
|
"sampling/sampling_logp_difference/max": 0.32407853603363035,
|
|
"sampling/sampling_logp_difference/mean": 0.021431529615074395,
|
|
"step": 4690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.2,
|
|
"completions/max_terminated_length": 291.2,
|
|
"completions/mean_length": 205.575,
|
|
"completions/mean_terminated_length": 205.575,
|
|
"completions/min_length": 131.7,
|
|
"completions/min_terminated_length": 131.7,
|
|
"entropy": 1.061689742654562,
|
|
"epoch": 0.4321044405626551,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.998540878295898,
|
|
"kl": 0.04379493425949477,
|
|
"learning_rate": 4.8802039189465164e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 20383642.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.32402134537696836,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.3240213692188263,
|
|
"sampling/importance_sampling_ratio/max": 1.2807414531707764,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003158509731294,
|
|
"sampling/importance_sampling_ratio/min": 0.7573738515377044,
|
|
"sampling/sampling_logp_difference/max": 0.29787570238113403,
|
|
"sampling/sampling_logp_difference/mean": 0.021973891928792,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.8,
|
|
"completions/max_terminated_length": 294.8,
|
|
"completions/mean_length": 197.5625,
|
|
"completions/mean_terminated_length": 197.5625,
|
|
"completions/min_length": 126.5,
|
|
"completions/min_terminated_length": 126.5,
|
|
"entropy": 1.1184421628713608,
|
|
"epoch": 0.4330238117127885,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.11805090308189392,
|
|
"kl": 0.03993665155721828,
|
|
"learning_rate": 4.879427802426829e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 20423959.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3601123929023743,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.36011239886283875,
|
|
"sampling/importance_sampling_ratio/max": 1.2567843079566956,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999920904636384,
|
|
"sampling/importance_sampling_ratio/min": 0.7700425803661346,
|
|
"sampling/sampling_logp_difference/max": 0.2839848279953003,
|
|
"sampling/sampling_logp_difference/mean": 0.022143154032528402,
|
|
"step": 4710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.7,
|
|
"completions/max_terminated_length": 296.7,
|
|
"completions/mean_length": 207.0375,
|
|
"completions/mean_terminated_length": 207.0375,
|
|
"completions/min_length": 118.1,
|
|
"completions/min_terminated_length": 118.1,
|
|
"entropy": 1.0847640939056873,
|
|
"epoch": 0.4339431828629218,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.291732311248779,
|
|
"kl": 0.04596530375420116,
|
|
"learning_rate": 4.878649242044055e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 20465938.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3745658665895462,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.37456587255001067,
|
|
"sampling/importance_sampling_ratio/max": 1.274799907207489,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002844333648682,
|
|
"sampling/importance_sampling_ratio/min": 0.77854705452919,
|
|
"sampling/sampling_logp_difference/max": 0.27930507659912107,
|
|
"sampling/sampling_logp_difference/mean": 0.021757892705500124,
|
|
"step": 4720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.9,
|
|
"completions/max_terminated_length": 273.9,
|
|
"completions/mean_length": 199.0875,
|
|
"completions/mean_terminated_length": 199.0875,
|
|
"completions/min_length": 140.1,
|
|
"completions/min_terminated_length": 140.1,
|
|
"entropy": 1.0461763590574265,
|
|
"epoch": 0.43486255401305507,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.0319013595581055,
|
|
"kl": 0.0356659253127873,
|
|
"learning_rate": 4.877868238597836e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 20508065.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.3090260297060013,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.3090260297060013,
|
|
"sampling/importance_sampling_ratio/max": 1.2430699944496155,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004068553447723,
|
|
"sampling/importance_sampling_ratio/min": 0.7657820165157319,
|
|
"sampling/sampling_logp_difference/max": 0.2779232501983643,
|
|
"sampling/sampling_logp_difference/mean": 0.021514142118394374,
|
|
"step": 4730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.5,
|
|
"completions/max_terminated_length": 316.5,
|
|
"completions/mean_length": 227.4375,
|
|
"completions/mean_terminated_length": 227.4375,
|
|
"completions/min_length": 127.0,
|
|
"completions/min_terminated_length": 127.0,
|
|
"entropy": 1.1788320749998094,
|
|
"epoch": 0.43578192516318837,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1087602823972702,
|
|
"kl": 0.04807308245217427,
|
|
"learning_rate": 4.877084792890327e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 20558324.0,
|
|
"reward": -0.34375,
|
|
"reward_std": 0.32785561978816985,
|
|
"rewards/reward_fn/mean": -0.34375,
|
|
"rewards/reward_fn/std": 0.32785562574863436,
|
|
"sampling/importance_sampling_ratio/max": 1.2695066571235656,
|
|
"sampling/importance_sampling_ratio/mean": 1.000443547964096,
|
|
"sampling/importance_sampling_ratio/min": 0.7891834795475006,
|
|
"sampling/sampling_logp_difference/max": 0.27142399549484253,
|
|
"sampling/sampling_logp_difference/mean": 0.0224968783557415,
|
|
"step": 4740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.6,
|
|
"completions/max_terminated_length": 288.6,
|
|
"completions/mean_length": 205.475,
|
|
"completions/mean_terminated_length": 205.475,
|
|
"completions/min_length": 123.7,
|
|
"completions/min_terminated_length": 123.7,
|
|
"entropy": 1.1261117294430734,
|
|
"epoch": 0.43670129631332166,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.507055282592773,
|
|
"kl": 0.050419921055436134,
|
|
"learning_rate": 4.876298905726189e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 20601690.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.3446744382381439,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.3446744501590729,
|
|
"sampling/importance_sampling_ratio/max": 1.251662278175354,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999021172523499,
|
|
"sampling/importance_sampling_ratio/min": 0.7818563878536224,
|
|
"sampling/sampling_logp_difference/max": 0.25606557726860046,
|
|
"sampling/sampling_logp_difference/mean": 0.022135718166828154,
|
|
"step": 4750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.3,
|
|
"completions/max_terminated_length": 325.3,
|
|
"completions/mean_length": 228.475,
|
|
"completions/mean_terminated_length": 228.475,
|
|
"completions/min_length": 149.5,
|
|
"completions/min_terminated_length": 149.5,
|
|
"entropy": 1.1918197348713875,
|
|
"epoch": 0.437620667463455,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.183346271514893,
|
|
"kl": 0.04097719464916736,
|
|
"learning_rate": 4.8755105779125915e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 20647776.0,
|
|
"reward": -0.2625,
|
|
"reward_std": 0.422067266702652,
|
|
"rewards/reward_fn/mean": -0.2625,
|
|
"rewards/reward_fn/std": 0.4220672965049744,
|
|
"sampling/importance_sampling_ratio/max": 1.3097922205924988,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999755859375,
|
|
"sampling/importance_sampling_ratio/min": 0.7744962632656097,
|
|
"sampling/sampling_logp_difference/max": 0.27636181712150576,
|
|
"sampling/sampling_logp_difference/mean": 0.022780642844736576,
|
|
"step": 4760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.8,
|
|
"completions/max_terminated_length": 315.8,
|
|
"completions/mean_length": 229.075,
|
|
"completions/mean_terminated_length": 229.075,
|
|
"completions/min_length": 151.6,
|
|
"completions/min_terminated_length": 151.6,
|
|
"entropy": 1.158103872090578,
|
|
"epoch": 0.4385400386135883,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.777389049530029,
|
|
"kl": 0.04606992704793811,
|
|
"learning_rate": 4.874719810259209e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 20692478.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.3722868412733078,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.37228684723377226,
|
|
"sampling/importance_sampling_ratio/max": 1.2318024516105652,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001806914806366,
|
|
"sampling/importance_sampling_ratio/min": 0.7900952875614167,
|
|
"sampling/sampling_logp_difference/max": 0.2426864504814148,
|
|
"sampling/sampling_logp_difference/mean": 0.021408633701503275,
|
|
"step": 4770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.5,
|
|
"completions/max_terminated_length": 314.5,
|
|
"completions/mean_length": 203.825,
|
|
"completions/mean_terminated_length": 203.825,
|
|
"completions/min_length": 118.0,
|
|
"completions/min_terminated_length": 118.0,
|
|
"entropy": 1.1198556058108806,
|
|
"epoch": 0.4394594097637216,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.2967608571052551,
|
|
"kl": 0.05348022039979696,
|
|
"learning_rate": 4.873926603578225e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 20736312.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.490106862783432,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.490106874704361,
|
|
"sampling/importance_sampling_ratio/max": 1.2574563026428223,
|
|
"sampling/importance_sampling_ratio/mean": 0.999538105726242,
|
|
"sampling/importance_sampling_ratio/min": 0.7676050066947937,
|
|
"sampling/sampling_logp_difference/max": 0.2833777368068695,
|
|
"sampling/sampling_logp_difference/mean": 0.02239927966147661,
|
|
"step": 4780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.4,
|
|
"completions/max_terminated_length": 314.4,
|
|
"completions/mean_length": 220.4875,
|
|
"completions/mean_terminated_length": 220.4875,
|
|
"completions/min_length": 129.9,
|
|
"completions/min_terminated_length": 129.9,
|
|
"entropy": 1.0699482671916485,
|
|
"epoch": 0.4403787809138549,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.024379253387451,
|
|
"kl": 0.034763780643697825,
|
|
"learning_rate": 4.873130958684325e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 20776679.0,
|
|
"reward": -0.01875,
|
|
"reward_std": 0.4183605372905731,
|
|
"rewards/reward_fn/mean": -0.01875,
|
|
"rewards/reward_fn/std": 0.4183605372905731,
|
|
"sampling/importance_sampling_ratio/max": 1.3236806988716125,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003363728523254,
|
|
"sampling/importance_sampling_ratio/min": 0.7648263692855835,
|
|
"sampling/sampling_logp_difference/max": 0.2967477381229401,
|
|
"sampling/sampling_logp_difference/mean": 0.021180013380944728,
|
|
"step": 4790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.2,
|
|
"completions/max_terminated_length": 289.2,
|
|
"completions/mean_length": 216.15,
|
|
"completions/mean_terminated_length": 216.15,
|
|
"completions/min_length": 143.4,
|
|
"completions/min_terminated_length": 143.4,
|
|
"entropy": 1.0816289708018303,
|
|
"epoch": 0.44129815206398826,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.013596534729004,
|
|
"kl": 0.0373613010160625,
|
|
"learning_rate": 4.872332876394701e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 20819579.0,
|
|
"reward": -0.375,
|
|
"reward_std": 0.38671192824840545,
|
|
"rewards/reward_fn/mean": -0.375,
|
|
"rewards/reward_fn/std": 0.38671195805072783,
|
|
"sampling/importance_sampling_ratio/max": 1.2334882140159606,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999730110168457,
|
|
"sampling/importance_sampling_ratio/min": 0.8005930781364441,
|
|
"sampling/sampling_logp_difference/max": 0.23208605647087097,
|
|
"sampling/sampling_logp_difference/mean": 0.02125703003257513,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"epoch": 0.44129815206398826,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0010907504363001745,
|
|
"eval_completions/max_length": 296.08202443280976,
|
|
"eval_completions/max_terminated_length": 295.1413612565445,
|
|
"eval_completions/mean_length": 209.43041012216406,
|
|
"eval_completions/mean_terminated_length": 209.15108462029102,
|
|
"eval_completions/min_length": 135.54101221640488,
|
|
"eval_completions/min_terminated_length": 135.54101221640488,
|
|
"eval_entropy": 1.1251758550786224,
|
|
"eval_frac_reward_zero_std": 0.2530541012216405,
|
|
"eval_kl": 0.036473345590095456,
|
|
"eval_loss": 0.0018871371867135167,
|
|
"eval_num_tokens": 20819579.0,
|
|
"eval_reward": 0.025632635253054103,
|
|
"eval_reward_std": 0.40977246084138363,
|
|
"eval_rewards/reward_fn/mean": 0.025632635253054103,
|
|
"eval_rewards/reward_fn/std": 0.4097724723878331,
|
|
"eval_runtime": 1730.3128,
|
|
"eval_samples_per_second": 0.331,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2654379200560884,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999881724828616,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7834413237804726,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2702022853322054,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021100410039213213,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.2,
|
|
"completions/max_terminated_length": 292.2,
|
|
"completions/mean_length": 191.3875,
|
|
"completions/mean_terminated_length": 191.3875,
|
|
"completions/min_length": 120.5,
|
|
"completions/min_terminated_length": 120.5,
|
|
"entropy": 1.1443199023604393,
|
|
"epoch": 0.44221752321412156,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.949553966522217,
|
|
"kl": 0.03462678255746141,
|
|
"learning_rate": 4.871532357529048e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 20860546.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.5418508559465408,
|
|
"rewards/reward_fn/mean": 0.375,
|
|
"rewards/reward_fn/std": 0.5418508738279343,
|
|
"sampling/importance_sampling_ratio/max": 1.298871600627899,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003972351551056,
|
|
"sampling/importance_sampling_ratio/min": 0.7718465030193329,
|
|
"sampling/sampling_logp_difference/max": 0.3070435106754303,
|
|
"sampling/sampling_logp_difference/mean": 0.022321824356913565,
|
|
"step": 4810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 323.7,
|
|
"completions/max_terminated_length": 316.1,
|
|
"completions/mean_length": 224.275,
|
|
"completions/mean_terminated_length": 221.37142944335938,
|
|
"completions/min_length": 152.9,
|
|
"completions/min_terminated_length": 152.9,
|
|
"entropy": 1.115078853815794,
|
|
"epoch": 0.44313689436425485,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.023745536804199,
|
|
"kl": 0.03538187982048839,
|
|
"learning_rate": 4.870729402909561e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 20904856.0,
|
|
"reward": -0.15625,
|
|
"reward_std": 0.454889839887619,
|
|
"rewards/reward_fn/mean": -0.15625,
|
|
"rewards/reward_fn/std": 0.45488986372947693,
|
|
"sampling/importance_sampling_ratio/max": 1.2468860387802123,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005113661289216,
|
|
"sampling/importance_sampling_ratio/min": 0.7849171757698059,
|
|
"sampling/sampling_logp_difference/max": 0.2617574095726013,
|
|
"sampling/sampling_logp_difference/mean": 0.021430672705173494,
|
|
"step": 4820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 353.8,
|
|
"completions/max_terminated_length": 339.5,
|
|
"completions/mean_length": 227.6875,
|
|
"completions/mean_terminated_length": 224.91607360839845,
|
|
"completions/min_length": 135.9,
|
|
"completions/min_terminated_length": 135.9,
|
|
"entropy": 1.1646183796226979,
|
|
"epoch": 0.44405626551438815,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.444431304931641,
|
|
"kl": 0.04444430838339031,
|
|
"learning_rate": 4.869924013360941e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 20949615.0,
|
|
"reward": -0.10625,
|
|
"reward_std": 0.47085602283477784,
|
|
"rewards/reward_fn/mean": -0.10625,
|
|
"rewards/reward_fn/std": 0.4708560347557068,
|
|
"sampling/importance_sampling_ratio/max": 1.236397647857666,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997000753879547,
|
|
"sampling/importance_sampling_ratio/min": 0.7919482290744781,
|
|
"sampling/sampling_logp_difference/max": 0.2479753702878952,
|
|
"sampling/sampling_logp_difference/mean": 0.021419833600521087,
|
|
"step": 4830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.2,
|
|
"completions/max_terminated_length": 261.2,
|
|
"completions/mean_length": 188.3375,
|
|
"completions/mean_terminated_length": 188.3375,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 1.1247911766171454,
|
|
"epoch": 0.44497563666452145,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.723733901977539,
|
|
"kl": 0.03544203532510437,
|
|
"learning_rate": 4.869116189710386e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 20995274.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.3544388711452484,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.3544388830661774,
|
|
"sampling/importance_sampling_ratio/max": 1.2171321392059327,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997370958328247,
|
|
"sampling/importance_sampling_ratio/min": 0.7929406702518463,
|
|
"sampling/sampling_logp_difference/max": 0.23929535150527953,
|
|
"sampling/sampling_logp_difference/mean": 0.021614062413573266,
|
|
"step": 4840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.8,
|
|
"completions/max_terminated_length": 304.8,
|
|
"completions/mean_length": 202.425,
|
|
"completions/mean_terminated_length": 202.425,
|
|
"completions/min_length": 137.6,
|
|
"completions/min_terminated_length": 137.6,
|
|
"entropy": 1.0920110821723938,
|
|
"epoch": 0.4458950078146548,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.844337463378906,
|
|
"kl": 0.04696904121665284,
|
|
"learning_rate": 4.868305932787596e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 21039276.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.3739841729402542,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.3739841729402542,
|
|
"sampling/importance_sampling_ratio/max": 1.3499962449073792,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996867001056671,
|
|
"sampling/importance_sampling_ratio/min": 0.7630034983158112,
|
|
"sampling/sampling_logp_difference/max": 0.34782400727272034,
|
|
"sampling/sampling_logp_difference/mean": 0.02176199834793806,
|
|
"step": 4850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.2,
|
|
"completions/max_terminated_length": 307.2,
|
|
"completions/mean_length": 230.9625,
|
|
"completions/mean_terminated_length": 230.9625,
|
|
"completions/min_length": 152.1,
|
|
"completions/min_terminated_length": 152.1,
|
|
"entropy": 1.1516052424907683,
|
|
"epoch": 0.4468143789647881,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.6868438720703125,
|
|
"kl": 0.050775047903880474,
|
|
"learning_rate": 4.8674932434247715e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 21088361.0,
|
|
"reward": -0.23125,
|
|
"reward_std": 0.43632136285305023,
|
|
"rewards/reward_fn/mean": -0.23125,
|
|
"rewards/reward_fn/std": 0.4363213688135147,
|
|
"sampling/importance_sampling_ratio/max": 1.3010708332061767,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998128712177277,
|
|
"sampling/importance_sampling_ratio/min": 0.7771371245384217,
|
|
"sampling/sampling_logp_difference/max": 0.2982410728931427,
|
|
"sampling/sampling_logp_difference/mean": 0.02164751309901476,
|
|
"step": 4860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 256.5,
|
|
"completions/max_terminated_length": 256.5,
|
|
"completions/mean_length": 188.1125,
|
|
"completions/mean_terminated_length": 188.1125,
|
|
"completions/min_length": 118.1,
|
|
"completions/min_terminated_length": 118.1,
|
|
"entropy": 1.1325742185115815,
|
|
"epoch": 0.4477337501149214,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.810248374938965,
|
|
"kl": 0.04936469867825508,
|
|
"learning_rate": 4.866678122456607e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 21126426.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.39796746969223024,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.3979674756526947,
|
|
"sampling/importance_sampling_ratio/max": 1.302277398109436,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005680859088897,
|
|
"sampling/importance_sampling_ratio/min": 0.7990141689777375,
|
|
"sampling/sampling_logp_difference/max": 0.26777092218399046,
|
|
"sampling/sampling_logp_difference/mean": 0.02273658588528633,
|
|
"step": 4870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.4,
|
|
"completions/max_terminated_length": 294.4,
|
|
"completions/mean_length": 203.8625,
|
|
"completions/mean_terminated_length": 203.8625,
|
|
"completions/min_length": 133.7,
|
|
"completions/min_terminated_length": 133.7,
|
|
"entropy": 1.1043013699352742,
|
|
"epoch": 0.4486531212650547,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.195316314697266,
|
|
"kl": 0.054411167721264066,
|
|
"learning_rate": 4.865860570720298e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 21169183.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4739816963672638,
|
|
"rewards/reward_fn/mean": 0.15625,
|
|
"rewards/reward_fn/std": 0.4739817142486572,
|
|
"sampling/importance_sampling_ratio/max": 1.3177698731422425,
|
|
"sampling/importance_sampling_ratio/mean": 1.0009174585342406,
|
|
"sampling/importance_sampling_ratio/min": 0.7824495315551758,
|
|
"sampling/sampling_logp_difference/max": 0.3104393482208252,
|
|
"sampling/sampling_logp_difference/mean": 0.021653312258422375,
|
|
"step": 4880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.5,
|
|
"completions/max_terminated_length": 279.5,
|
|
"completions/mean_length": 197.5125,
|
|
"completions/mean_terminated_length": 197.5125,
|
|
"completions/min_length": 136.3,
|
|
"completions/min_terminated_length": 136.3,
|
|
"entropy": 1.1416750863194465,
|
|
"epoch": 0.449572492415188,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.275676250457764,
|
|
"kl": 0.06414212052477523,
|
|
"learning_rate": 4.865040589055537e-06,
|
|
"loss": 0.0035,
|
|
"num_tokens": 21210336.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.3291133403778076,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.3291133463382721,
|
|
"sampling/importance_sampling_ratio/max": 1.251700758934021,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004221558570863,
|
|
"sampling/importance_sampling_ratio/min": 0.7801296293735505,
|
|
"sampling/sampling_logp_difference/max": 0.2666925609111786,
|
|
"sampling/sampling_logp_difference/mean": 0.02234300263226032,
|
|
"step": 4890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.6,
|
|
"completions/max_terminated_length": 282.6,
|
|
"completions/mean_length": 206.125,
|
|
"completions/mean_terminated_length": 206.125,
|
|
"completions/min_length": 143.2,
|
|
"completions/min_terminated_length": 143.2,
|
|
"entropy": 0.9738481394946575,
|
|
"epoch": 0.45049186356532134,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.883378505706787,
|
|
"kl": 0.06306481685605832,
|
|
"learning_rate": 4.86421817830451e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 21251994.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37551296651363375,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.37551297843456266,
|
|
"sampling/importance_sampling_ratio/max": 1.2437256813049316,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002299964427948,
|
|
"sampling/importance_sampling_ratio/min": 0.7948164463043212,
|
|
"sampling/sampling_logp_difference/max": 0.24511125087738037,
|
|
"sampling/sampling_logp_difference/mean": 0.019641385413706303,
|
|
"step": 4900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 249.8,
|
|
"completions/max_terminated_length": 249.8,
|
|
"completions/mean_length": 185.4875,
|
|
"completions/mean_terminated_length": 185.4875,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 1.0819780327379704,
|
|
"epoch": 0.45141123471545463,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.20682430267334,
|
|
"kl": 0.050309620436746624,
|
|
"learning_rate": 4.863393339311897e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 21292905.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.5635587096214294,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.5635587275028229,
|
|
"sampling/importance_sampling_ratio/max": 1.276726531982422,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999016582965851,
|
|
"sampling/importance_sampling_ratio/min": 0.7862579941749572,
|
|
"sampling/sampling_logp_difference/max": 0.27232807874679565,
|
|
"sampling/sampling_logp_difference/mean": 0.020635491982102395,
|
|
"step": 4910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.0,
|
|
"completions/max_terminated_length": 313.0,
|
|
"completions/mean_length": 226.3375,
|
|
"completions/mean_terminated_length": 226.3375,
|
|
"completions/min_length": 156.6,
|
|
"completions/min_terminated_length": 156.6,
|
|
"entropy": 1.1471611127257346,
|
|
"epoch": 0.45233060586558793,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.578601837158203,
|
|
"kl": 0.041793910751584916,
|
|
"learning_rate": 4.8625660729248765e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 21338524.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.33258684873580935,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.3325868546962738,
|
|
"sampling/importance_sampling_ratio/max": 1.2573898077011108,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996989667415619,
|
|
"sampling/importance_sampling_ratio/min": 0.7774382889270782,
|
|
"sampling/sampling_logp_difference/max": 0.28128833770751954,
|
|
"sampling/sampling_logp_difference/mean": 0.021383467689156534,
|
|
"step": 4920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 337.3,
|
|
"completions/max_terminated_length": 337.3,
|
|
"completions/mean_length": 225.2625,
|
|
"completions/mean_terminated_length": 225.2625,
|
|
"completions/min_length": 119.3,
|
|
"completions/min_terminated_length": 119.3,
|
|
"entropy": 1.0893308449536563,
|
|
"epoch": 0.45324997701572123,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.952234745025635,
|
|
"kl": 0.0390682342054788,
|
|
"learning_rate": 4.861736379993116e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 21386161.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.3898056745529175,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.3898056924343109,
|
|
"sampling/importance_sampling_ratio/max": 1.303667378425598,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997180998325348,
|
|
"sampling/importance_sampling_ratio/min": 0.7679412007331848,
|
|
"sampling/sampling_logp_difference/max": 0.29677664041519164,
|
|
"sampling/sampling_logp_difference/mean": 0.02083776155486703,
|
|
"step": 4930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.3,
|
|
"completions/max_terminated_length": 302.3,
|
|
"completions/mean_length": 203.2375,
|
|
"completions/mean_terminated_length": 203.2375,
|
|
"completions/min_length": 104.6,
|
|
"completions/min_terminated_length": 104.6,
|
|
"entropy": 1.118368787318468,
|
|
"epoch": 0.4541693481658546,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.013526439666748,
|
|
"kl": 0.03668456432060339,
|
|
"learning_rate": 4.8609042613687766e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 21428508.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.3771726757287979,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.3771726995706558,
|
|
"sampling/importance_sampling_ratio/max": 1.3006525754928588,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996533751487732,
|
|
"sampling/importance_sampling_ratio/min": 0.8079426527023316,
|
|
"sampling/sampling_logp_difference/max": 0.26807049512863157,
|
|
"sampling/sampling_logp_difference/mean": 0.021350833773612975,
|
|
"step": 4940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.3,
|
|
"completions/max_terminated_length": 298.3,
|
|
"completions/mean_length": 200.075,
|
|
"completions/mean_terminated_length": 200.075,
|
|
"completions/min_length": 127.7,
|
|
"completions/min_terminated_length": 127.7,
|
|
"entropy": 1.011567797511816,
|
|
"epoch": 0.4550887193159879,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.646734237670898,
|
|
"kl": 0.033018376323161645,
|
|
"learning_rate": 4.860069717906511e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 21465562.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.2966861009597778,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.29668611884117124,
|
|
"sampling/importance_sampling_ratio/max": 1.2530601263046264,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997290253639222,
|
|
"sampling/importance_sampling_ratio/min": 0.7835807919502258,
|
|
"sampling/sampling_logp_difference/max": 0.2518495798110962,
|
|
"sampling/sampling_logp_difference/mean": 0.02033227076753974,
|
|
"step": 4950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.9,
|
|
"completions/max_terminated_length": 288.9,
|
|
"completions/mean_length": 193.325,
|
|
"completions/mean_terminated_length": 193.325,
|
|
"completions/min_length": 110.4,
|
|
"completions/min_terminated_length": 110.4,
|
|
"entropy": 1.1031865365803242,
|
|
"epoch": 0.4560080904661212,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.2061709761619568,
|
|
"kl": 0.03829234721488319,
|
|
"learning_rate": 4.859232750463463e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 21505996.0,
|
|
"reward": -0.4125,
|
|
"reward_std": 0.30762961506843567,
|
|
"rewards/reward_fn/mean": -0.4125,
|
|
"rewards/reward_fn/std": 0.3076296210289001,
|
|
"sampling/importance_sampling_ratio/max": 1.2420978546142578,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997165083885193,
|
|
"sampling/importance_sampling_ratio/min": 0.771349960565567,
|
|
"sampling/sampling_logp_difference/max": 0.26861617565155027,
|
|
"sampling/sampling_logp_difference/mean": 0.021324186585843563,
|
|
"step": 4960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.3,
|
|
"completions/max_terminated_length": 283.3,
|
|
"completions/mean_length": 200.5625,
|
|
"completions/mean_terminated_length": 200.5625,
|
|
"completions/min_length": 106.5,
|
|
"completions/min_terminated_length": 106.5,
|
|
"entropy": 1.1076211370527744,
|
|
"epoch": 0.45692746161625447,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.930084705352783,
|
|
"kl": 0.051061311236117034,
|
|
"learning_rate": 4.858393359899265e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 21548369.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.5144637852907181,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.5144638031721115,
|
|
"sampling/importance_sampling_ratio/max": 1.2712450623512268,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993958890438079,
|
|
"sampling/importance_sampling_ratio/min": 0.7928924083709716,
|
|
"sampling/sampling_logp_difference/max": 0.2608424425125122,
|
|
"sampling/sampling_logp_difference/mean": 0.021413145028054713,
|
|
"step": 4970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.1,
|
|
"completions/max_terminated_length": 306.1,
|
|
"completions/mean_length": 204.1,
|
|
"completions/mean_terminated_length": 204.1,
|
|
"completions/min_length": 117.1,
|
|
"completions/min_terminated_length": 117.1,
|
|
"entropy": 1.0390336617827416,
|
|
"epoch": 0.45784683276638777,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12613804638385773,
|
|
"kl": 0.04744800207554363,
|
|
"learning_rate": 4.857551547076037e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 21590177.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.28112500309944155,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.28112501502037046,
|
|
"sampling/importance_sampling_ratio/max": 1.2713823318481445,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003105998039246,
|
|
"sampling/importance_sampling_ratio/min": 0.785928875207901,
|
|
"sampling/sampling_logp_difference/max": 0.2674295246601105,
|
|
"sampling/sampling_logp_difference/mean": 0.020596507750451565,
|
|
"step": 4980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.5,
|
|
"completions/max_terminated_length": 313.5,
|
|
"completions/mean_length": 223.2125,
|
|
"completions/mean_terminated_length": 223.2125,
|
|
"completions/min_length": 118.8,
|
|
"completions/min_terminated_length": 118.8,
|
|
"entropy": 1.081412636488676,
|
|
"epoch": 0.4587662039165211,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.8120036125183105,
|
|
"kl": 0.044370586139848456,
|
|
"learning_rate": 4.85670731285839e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 21636170.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.3574239045381546,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.3574239045381546,
|
|
"sampling/importance_sampling_ratio/max": 1.2946985483169555,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002689242362977,
|
|
"sampling/importance_sampling_ratio/min": 0.7857405126094819,
|
|
"sampling/sampling_logp_difference/max": 0.29253565073013305,
|
|
"sampling/sampling_logp_difference/mean": 0.020496607571840287,
|
|
"step": 4990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.3,
|
|
"completions/max_terminated_length": 294.3,
|
|
"completions/mean_length": 208.95,
|
|
"completions/mean_terminated_length": 208.95,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"entropy": 1.1115488819777966,
|
|
"epoch": 0.4596855750666544,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.16221123933792114,
|
|
"kl": 0.05902964059496298,
|
|
"learning_rate": 4.855860658113419e-06,
|
|
"loss": 0.0034,
|
|
"num_tokens": 21678390.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.4237645983695984,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.4237646222114563,
|
|
"sampling/importance_sampling_ratio/max": 1.3044022798538208,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994701564311981,
|
|
"sampling/importance_sampling_ratio/min": 0.7895667552947998,
|
|
"sampling/sampling_logp_difference/max": 0.28730525970458987,
|
|
"sampling/sampling_logp_difference/mean": 0.02126925028860569,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.4596855750666544,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.001963350785340314,
|
|
"eval_completions/max_length": 296.6020942408377,
|
|
"eval_completions/max_terminated_length": 295.61605584642234,
|
|
"eval_completions/mean_length": 204.22687609075044,
|
|
"eval_completions/mean_terminated_length": 203.81525819255836,
|
|
"eval_completions/min_length": 123.58638743455498,
|
|
"eval_completions/min_terminated_length": 123.58638743455498,
|
|
"eval_entropy": 1.0980207964283841,
|
|
"eval_frac_reward_zero_std": 0.31239092495637,
|
|
"eval_kl": 0.05347816798556433,
|
|
"eval_loss": 0.0028688579332083464,
|
|
"eval_num_tokens": 21678390.0,
|
|
"eval_reward": 0.05606457242582897,
|
|
"eval_reward_std": 0.33468144091844143,
|
|
"eval_rewards/reward_fn/mean": 0.05606457242582897,
|
|
"eval_rewards/reward_fn/std": 0.33468145017640544,
|
|
"eval_runtime": 1729.3611,
|
|
"eval_samples_per_second": 0.331,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2829591175232467,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000462963735037,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7834602964174061,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2806670146150739,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02100302376028599,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.1,
|
|
"completions/max_terminated_length": 283.1,
|
|
"completions/mean_length": 215.05,
|
|
"completions/mean_terminated_length": 215.05,
|
|
"completions/min_length": 153.1,
|
|
"completions/min_terminated_length": 153.1,
|
|
"entropy": 1.0642635196447372,
|
|
"epoch": 0.4606049462167877,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.21094852685928345,
|
|
"kl": 0.054026177397463473,
|
|
"learning_rate": 4.855011583710707e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 21721586.0,
|
|
"reward": -0.00625,
|
|
"reward_std": 0.25884227454662323,
|
|
"rewards/reward_fn/mean": -0.00625,
|
|
"rewards/reward_fn/std": 0.25884227454662323,
|
|
"sampling/importance_sampling_ratio/max": 1.2514773607254028,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000546872615814,
|
|
"sampling/importance_sampling_ratio/min": 0.8022564351558685,
|
|
"sampling/sampling_logp_difference/max": 0.2431874692440033,
|
|
"sampling/sampling_logp_difference/mean": 0.021205791831016542,
|
|
"step": 5010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.1,
|
|
"completions/max_terminated_length": 291.1,
|
|
"completions/mean_length": 209.7625,
|
|
"completions/mean_terminated_length": 209.7625,
|
|
"completions/min_length": 135.4,
|
|
"completions/min_terminated_length": 135.4,
|
|
"entropy": 1.1110947892069816,
|
|
"epoch": 0.461524317366921,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.1219000816345215,
|
|
"kl": 0.055259722052142024,
|
|
"learning_rate": 4.854160090522321e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 21762631.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.3429849535226822,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.34298495948314667,
|
|
"sampling/importance_sampling_ratio/max": 1.2838879108428956,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997478723526001,
|
|
"sampling/importance_sampling_ratio/min": 0.7638672709465026,
|
|
"sampling/sampling_logp_difference/max": 0.30507512092590333,
|
|
"sampling/sampling_logp_difference/mean": 0.02249348033219576,
|
|
"step": 5020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 263.9,
|
|
"completions/max_terminated_length": 263.9,
|
|
"completions/mean_length": 186.8375,
|
|
"completions/mean_terminated_length": 186.8375,
|
|
"completions/min_length": 130.3,
|
|
"completions/min_terminated_length": 130.3,
|
|
"entropy": 1.1042222172021865,
|
|
"epoch": 0.4624436885170543,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.015363693237305,
|
|
"kl": 0.051652857108274476,
|
|
"learning_rate": 4.853306179422814e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 21804634.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.3538378983736038,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.35383791029453276,
|
|
"sampling/importance_sampling_ratio/max": 1.2638363718986512,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997810602188111,
|
|
"sampling/importance_sampling_ratio/min": 0.791809868812561,
|
|
"sampling/sampling_logp_difference/max": 0.2542734980583191,
|
|
"sampling/sampling_logp_difference/mean": 0.02173410803079605,
|
|
"step": 5030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.2,
|
|
"completions/max_terminated_length": 273.2,
|
|
"completions/mean_length": 180.925,
|
|
"completions/mean_terminated_length": 180.925,
|
|
"completions/min_length": 109.1,
|
|
"completions/min_terminated_length": 109.1,
|
|
"entropy": 1.0299954634159803,
|
|
"epoch": 0.46336305966718766,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.978346347808838,
|
|
"kl": 0.06670718252426014,
|
|
"learning_rate": 4.852449851289219e-06,
|
|
"loss": 0.0034,
|
|
"num_tokens": 21847108.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.3946571469306946,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.394657164812088,
|
|
"sampling/importance_sampling_ratio/max": 1.288669967651367,
|
|
"sampling/importance_sampling_ratio/mean": 0.99981968998909,
|
|
"sampling/importance_sampling_ratio/min": 0.7698420405387878,
|
|
"sampling/sampling_logp_difference/max": 0.3021678805351257,
|
|
"sampling/sampling_logp_difference/mean": 0.02039196193218231,
|
|
"step": 5040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.0,
|
|
"completions/max_terminated_length": 278.0,
|
|
"completions/mean_length": 202.65,
|
|
"completions/mean_terminated_length": 202.65,
|
|
"completions/min_length": 115.6,
|
|
"completions/min_terminated_length": 115.6,
|
|
"entropy": 1.121798100322485,
|
|
"epoch": 0.46428243081732096,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.911308765411377,
|
|
"kl": 0.05855126996757463,
|
|
"learning_rate": 4.851591107001055e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 21889848.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.40020377933979034,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.40020379722118377,
|
|
"sampling/importance_sampling_ratio/max": 1.2433643817901612,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993038415908814,
|
|
"sampling/importance_sampling_ratio/min": 0.7627947509288788,
|
|
"sampling/sampling_logp_difference/max": 0.2757129192352295,
|
|
"sampling/sampling_logp_difference/mean": 0.022194026969373226,
|
|
"step": 5050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 264.8,
|
|
"completions/max_terminated_length": 264.8,
|
|
"completions/mean_length": 190.275,
|
|
"completions/mean_terminated_length": 190.275,
|
|
"completions/min_length": 94.1,
|
|
"completions/min_terminated_length": 94.1,
|
|
"entropy": 0.9969781950116158,
|
|
"epoch": 0.46520180196745425,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.016033172607422,
|
|
"kl": 0.04298663109075278,
|
|
"learning_rate": 4.850729947440321e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 21930542.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.3147830098867416,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.31478302776813505,
|
|
"sampling/importance_sampling_ratio/max": 1.313794457912445,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997335433959961,
|
|
"sampling/importance_sampling_ratio/min": 0.7648922443389893,
|
|
"sampling/sampling_logp_difference/max": 0.31616237163543703,
|
|
"sampling/sampling_logp_difference/mean": 0.02098549772053957,
|
|
"step": 5060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 332.4,
|
|
"completions/max_terminated_length": 310.8,
|
|
"completions/mean_length": 235.4625,
|
|
"completions/mean_terminated_length": 229.71607360839843,
|
|
"completions/min_length": 117.9,
|
|
"completions/min_terminated_length": 117.9,
|
|
"entropy": 1.0497488506138324,
|
|
"epoch": 0.46612117311758755,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1476282775402069,
|
|
"kl": 0.04426348961424083,
|
|
"learning_rate": 4.8498663734914975e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 21974763.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.41783192455768586,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.41783193647861483,
|
|
"sampling/importance_sampling_ratio/max": 1.2631876111030578,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993622124195098,
|
|
"sampling/importance_sampling_ratio/min": 0.783275592327118,
|
|
"sampling/sampling_logp_difference/max": 0.2601447343826294,
|
|
"sampling/sampling_logp_difference/mean": 0.021352509409189223,
|
|
"step": 5070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.5,
|
|
"completions/max_terminated_length": 299.5,
|
|
"completions/mean_length": 198.7,
|
|
"completions/mean_terminated_length": 198.7,
|
|
"completions/min_length": 119.7,
|
|
"completions/min_terminated_length": 119.7,
|
|
"entropy": 1.1181830674409867,
|
|
"epoch": 0.4670405442677209,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.18036657571792603,
|
|
"kl": 0.054512911546044054,
|
|
"learning_rate": 4.8490003860415436e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 22015267.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.3348544090986252,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.3348544090986252,
|
|
"sampling/importance_sampling_ratio/max": 1.2961976051330566,
|
|
"sampling/importance_sampling_ratio/mean": 1.000363713502884,
|
|
"sampling/importance_sampling_ratio/min": 0.7825570344924927,
|
|
"sampling/sampling_logp_difference/max": 0.2936065196990967,
|
|
"sampling/sampling_logp_difference/mean": 0.022312752902507782,
|
|
"step": 5080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.3,
|
|
"completions/max_terminated_length": 282.3,
|
|
"completions/mean_length": 200.85,
|
|
"completions/mean_terminated_length": 200.85,
|
|
"completions/min_length": 130.6,
|
|
"completions/min_terminated_length": 130.6,
|
|
"entropy": 1.059295652806759,
|
|
"epoch": 0.4679599154178542,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.634043216705322,
|
|
"kl": 0.04717031229520217,
|
|
"learning_rate": 4.848131985979897e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 22059839.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.3302692323923111,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.3302692502737045,
|
|
"sampling/importance_sampling_ratio/max": 1.3089182257652283,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001337468624114,
|
|
"sampling/importance_sampling_ratio/min": 0.7484601140022278,
|
|
"sampling/sampling_logp_difference/max": 0.3212741196155548,
|
|
"sampling/sampling_logp_difference/mean": 0.02130128536373377,
|
|
"step": 5090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 266.0,
|
|
"completions/max_terminated_length": 266.0,
|
|
"completions/mean_length": 182.15,
|
|
"completions/mean_terminated_length": 182.15,
|
|
"completions/min_length": 112.6,
|
|
"completions/min_terminated_length": 112.6,
|
|
"entropy": 1.0159906573593616,
|
|
"epoch": 0.4688792865679875,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.790402889251709,
|
|
"kl": 0.03802815774688497,
|
|
"learning_rate": 4.847261174198475e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 22104979.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.26302809417247774,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.26302810609340666,
|
|
"sampling/importance_sampling_ratio/max": 1.2474716305732727,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999119460582733,
|
|
"sampling/importance_sampling_ratio/min": 0.8018891036510467,
|
|
"sampling/sampling_logp_difference/max": 0.2434978485107422,
|
|
"sampling/sampling_logp_difference/mean": 0.020131850615143776,
|
|
"step": 5100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 263.4,
|
|
"completions/max_terminated_length": 263.4,
|
|
"completions/mean_length": 185.8375,
|
|
"completions/mean_terminated_length": 185.8375,
|
|
"completions/min_length": 129.7,
|
|
"completions/min_terminated_length": 129.7,
|
|
"entropy": 1.0033727928996086,
|
|
"epoch": 0.4697986577181208,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.781394958496094,
|
|
"kl": 0.04538027112139389,
|
|
"learning_rate": 4.846387951591672e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 22146982.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.42685834467411043,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.42685835659503935,
|
|
"sampling/importance_sampling_ratio/max": 1.2272173047065735,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000843107700348,
|
|
"sampling/importance_sampling_ratio/min": 0.8051585793495178,
|
|
"sampling/sampling_logp_difference/max": 0.2425686836242676,
|
|
"sampling/sampling_logp_difference/mean": 0.020243636891245843,
|
|
"step": 5110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.0,
|
|
"completions/max_terminated_length": 316.0,
|
|
"completions/mean_length": 208.275,
|
|
"completions/mean_terminated_length": 208.275,
|
|
"completions/min_length": 118.0,
|
|
"completions/min_terminated_length": 118.0,
|
|
"entropy": 1.0271643541753293,
|
|
"epoch": 0.4707180288682541,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.1978535652160645,
|
|
"kl": 0.04922398622147739,
|
|
"learning_rate": 4.8455123190563556e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 22188748.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.37944808304309846,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.3794480890035629,
|
|
"sampling/importance_sampling_ratio/max": 1.2611426711082458,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002716302871704,
|
|
"sampling/importance_sampling_ratio/min": 0.732643648982048,
|
|
"sampling/sampling_logp_difference/max": 0.3451880097389221,
|
|
"sampling/sampling_logp_difference/mean": 0.02061178386211395,
|
|
"step": 5120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.3,
|
|
"completions/max_terminated_length": 283.3,
|
|
"completions/mean_length": 197.725,
|
|
"completions/mean_terminated_length": 197.725,
|
|
"completions/min_length": 122.4,
|
|
"completions/min_terminated_length": 122.4,
|
|
"entropy": 1.0985075183212758,
|
|
"epoch": 0.47163740001838744,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.584001064300537,
|
|
"kl": 0.0392339939950034,
|
|
"learning_rate": 4.844634277491873e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 22231718.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3909654438495636,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.3909654557704926,
|
|
"sampling/importance_sampling_ratio/max": 1.3278156399726868,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000971794128417,
|
|
"sampling/importance_sampling_ratio/min": 0.7678834438323975,
|
|
"sampling/sampling_logp_difference/max": 0.3281996488571167,
|
|
"sampling/sampling_logp_difference/mean": 0.02178528197109699,
|
|
"step": 5130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.2,
|
|
"completions/max_terminated_length": 268.2,
|
|
"completions/mean_length": 181.925,
|
|
"completions/mean_terminated_length": 181.925,
|
|
"completions/min_length": 116.9,
|
|
"completions/min_terminated_length": 116.9,
|
|
"entropy": 1.101681013405323,
|
|
"epoch": 0.47255677116852074,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.21135449409484863,
|
|
"kl": 0.046971906477119775,
|
|
"learning_rate": 4.843753827800041e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 22273152.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.28029437065124513,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.28029437065124513,
|
|
"sampling/importance_sampling_ratio/max": 1.273150885105133,
|
|
"sampling/importance_sampling_ratio/mean": 1.000169610977173,
|
|
"sampling/importance_sampling_ratio/min": 0.78610959649086,
|
|
"sampling/sampling_logp_difference/max": 0.26658374071121216,
|
|
"sampling/sampling_logp_difference/mean": 0.021707371808588505,
|
|
"step": 5140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.9,
|
|
"completions/max_terminated_length": 289.9,
|
|
"completions/mean_length": 201.0625,
|
|
"completions/mean_terminated_length": 201.0625,
|
|
"completions/min_length": 124.3,
|
|
"completions/min_terminated_length": 124.3,
|
|
"entropy": 1.081572512537241,
|
|
"epoch": 0.47347614231865404,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.036871433258057,
|
|
"kl": 0.05442186284344643,
|
|
"learning_rate": 4.842870970885153e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 22317069.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.37833244502544405,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.3783324629068375,
|
|
"sampling/importance_sampling_ratio/max": 1.242033851146698,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998442113399506,
|
|
"sampling/importance_sampling_ratio/min": 0.7473058223724365,
|
|
"sampling/sampling_logp_difference/max": 0.2957075357437134,
|
|
"sampling/sampling_logp_difference/mean": 0.021647975780069828,
|
|
"step": 5150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 243.1,
|
|
"completions/max_terminated_length": 243.1,
|
|
"completions/mean_length": 175.5125,
|
|
"completions/mean_terminated_length": 175.5125,
|
|
"completions/min_length": 116.7,
|
|
"completions/min_terminated_length": 116.7,
|
|
"entropy": 1.0082119680941104,
|
|
"epoch": 0.47439551346878733,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.256570816040039,
|
|
"kl": 0.05352669929852709,
|
|
"learning_rate": 4.841985707653974e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 22355974.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.3863084077835083,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.3863084137439728,
|
|
"sampling/importance_sampling_ratio/max": 1.2449208736419677,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003450989723206,
|
|
"sampling/importance_sampling_ratio/min": 0.7393296301364899,
|
|
"sampling/sampling_logp_difference/max": 0.3140349149703979,
|
|
"sampling/sampling_logp_difference/mean": 0.02000609301030636,
|
|
"step": 5160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.8,
|
|
"completions/max_terminated_length": 308.8,
|
|
"completions/mean_length": 208.75,
|
|
"completions/mean_terminated_length": 208.75,
|
|
"completions/min_length": 138.0,
|
|
"completions/min_terminated_length": 138.0,
|
|
"entropy": 1.1051113750785588,
|
|
"epoch": 0.4753148846189207,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.527437686920166,
|
|
"kl": 0.03379192901775241,
|
|
"learning_rate": 4.84109803901574e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 22399602.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.390112441778183,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.39011244773864745,
|
|
"sampling/importance_sampling_ratio/max": 1.2987023949623109,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995889306068421,
|
|
"sampling/importance_sampling_ratio/min": 0.7780885815620422,
|
|
"sampling/sampling_logp_difference/max": 0.2858575820922852,
|
|
"sampling/sampling_logp_difference/mean": 0.021906653419137,
|
|
"step": 5170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.0,
|
|
"completions/max_terminated_length": 290.0,
|
|
"completions/mean_length": 191.5875,
|
|
"completions/mean_terminated_length": 191.5875,
|
|
"completions/min_length": 110.5,
|
|
"completions/min_terminated_length": 110.5,
|
|
"entropy": 1.1301247969269752,
|
|
"epoch": 0.476234255769054,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.14744135737419128,
|
|
"kl": 0.04660897683352232,
|
|
"learning_rate": 4.840207965882158e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 22440129.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.4498067259788513,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.44980673789978026,
|
|
"sampling/importance_sampling_ratio/max": 1.2783320546150208,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997207224369049,
|
|
"sampling/importance_sampling_ratio/min": 0.7876246988773346,
|
|
"sampling/sampling_logp_difference/max": 0.27246793508529665,
|
|
"sampling/sampling_logp_difference/mean": 0.022337763011455535,
|
|
"step": 5180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.2,
|
|
"completions/max_terminated_length": 283.2,
|
|
"completions/mean_length": 217.7,
|
|
"completions/mean_terminated_length": 217.7,
|
|
"completions/min_length": 131.4,
|
|
"completions/min_terminated_length": 131.4,
|
|
"entropy": 1.0966159895062446,
|
|
"epoch": 0.4771536269191873,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.527858734130859,
|
|
"kl": 0.048502184508834034,
|
|
"learning_rate": 4.839315489167405e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 22482289.0,
|
|
"reward": -0.225,
|
|
"reward_std": 0.4447881907224655,
|
|
"rewards/reward_fn/mean": -0.225,
|
|
"rewards/reward_fn/std": 0.4447882086038589,
|
|
"sampling/importance_sampling_ratio/max": 1.2794241547584533,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999913692474365,
|
|
"sampling/importance_sampling_ratio/min": 0.7823644757270813,
|
|
"sampling/sampling_logp_difference/max": 0.2830728769302368,
|
|
"sampling/sampling_logp_difference/mean": 0.02094872798770666,
|
|
"step": 5190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.5,
|
|
"completions/max_terminated_length": 297.5,
|
|
"completions/mean_length": 206.6625,
|
|
"completions/mean_terminated_length": 206.6625,
|
|
"completions/min_length": 138.8,
|
|
"completions/min_terminated_length": 138.8,
|
|
"entropy": 1.0814496617764235,
|
|
"epoch": 0.4780729980693206,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.223296165466309,
|
|
"kl": 0.038792222726624456,
|
|
"learning_rate": 4.838420609788125e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 22526494.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.45408297181129453,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.4540829837322235,
|
|
"sampling/importance_sampling_ratio/max": 1.3048575401306153,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005558133125305,
|
|
"sampling/importance_sampling_ratio/min": 0.7743797719478607,
|
|
"sampling/sampling_logp_difference/max": 0.29603824615478513,
|
|
"sampling/sampling_logp_difference/mean": 0.021037834510207177,
|
|
"step": 5200
|
|
},
|
|
{
|
|
"epoch": 0.4780729980693206,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.001963350785340314,
|
|
"eval_completions/max_length": 306.6020942408377,
|
|
"eval_completions/max_terminated_length": 304.95462478184993,
|
|
"eval_completions/mean_length": 215.4962914485166,
|
|
"eval_completions/mean_terminated_length": 215.05709314263095,
|
|
"eval_completions/min_length": 140.19197207678883,
|
|
"eval_completions/min_terminated_length": 140.19197207678883,
|
|
"eval_entropy": 1.136109718841616,
|
|
"eval_frac_reward_zero_std": 0.23036649214659685,
|
|
"eval_kl": 0.03790728024078653,
|
|
"eval_loss": 0.0019444205099716783,
|
|
"eval_num_tokens": 22526494.0,
|
|
"eval_reward": 0.035231239092495634,
|
|
"eval_reward_std": 0.41636209993462286,
|
|
"eval_rewards/reward_fn/mean": 0.035231239092495634,
|
|
"eval_rewards/reward_fn/std": 0.4163621130414034,
|
|
"eval_runtime": 1782.7889,
|
|
"eval_samples_per_second": 0.321,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2755815710280787,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000133960658968,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7840934941645901,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27744009159830435,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02109494226713247,
|
|
"eval_steps_per_second": 0.04,
|
|
"step": 5200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.7,
|
|
"completions/max_terminated_length": 289.7,
|
|
"completions/mean_length": 200.8375,
|
|
"completions/mean_terminated_length": 200.8375,
|
|
"completions/min_length": 143.7,
|
|
"completions/min_terminated_length": 143.7,
|
|
"entropy": 1.1420616939663888,
|
|
"epoch": 0.4789923692194539,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.956714630126953,
|
|
"kl": 0.049311512301210315,
|
|
"learning_rate": 4.8375233286634315e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 22569065.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.4520098477602005,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.45200985968112944,
|
|
"sampling/importance_sampling_ratio/max": 1.2439144253730774,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002358198165893,
|
|
"sampling/importance_sampling_ratio/min": 0.7658241093158722,
|
|
"sampling/sampling_logp_difference/max": 0.27454921007156374,
|
|
"sampling/sampling_logp_difference/mean": 0.021910637430846692,
|
|
"step": 5210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 333.0,
|
|
"completions/max_terminated_length": 333.0,
|
|
"completions/mean_length": 237.0,
|
|
"completions/mean_terminated_length": 237.0,
|
|
"completions/min_length": 146.3,
|
|
"completions/min_terminated_length": 146.3,
|
|
"entropy": 1.2142364338040352,
|
|
"epoch": 0.4799117403695872,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.120321273803711,
|
|
"kl": 0.039680343423970046,
|
|
"learning_rate": 4.836623646714904e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 22615033.0,
|
|
"reward": -0.225,
|
|
"reward_std": 0.3110164314508438,
|
|
"rewards/reward_fn/mean": -0.225,
|
|
"rewards/reward_fn/std": 0.3110164374113083,
|
|
"sampling/importance_sampling_ratio/max": 1.2647359848022461,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999931931495667,
|
|
"sampling/importance_sampling_ratio/min": 0.7919558465480805,
|
|
"sampling/sampling_logp_difference/max": 0.25625980496406553,
|
|
"sampling/sampling_logp_difference/mean": 0.0225582379847765,
|
|
"step": 5220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 338.3,
|
|
"completions/max_terminated_length": 320.9,
|
|
"completions/mean_length": 224.5375,
|
|
"completions/mean_terminated_length": 220.97857208251952,
|
|
"completions/min_length": 137.3,
|
|
"completions/min_terminated_length": 137.3,
|
|
"entropy": 1.1301677040755749,
|
|
"epoch": 0.4808311115197205,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.9213457107543945,
|
|
"kl": 0.0445600493112579,
|
|
"learning_rate": 4.8357215648665876e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 22660356.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.5354843109846115,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.5354843288660049,
|
|
"sampling/importance_sampling_ratio/max": 1.2766311645507813,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998451352119446,
|
|
"sampling/importance_sampling_ratio/min": 0.7980450689792633,
|
|
"sampling/sampling_logp_difference/max": 0.25270800590515136,
|
|
"sampling/sampling_logp_difference/mean": 0.02100846953690052,
|
|
"step": 5230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 338.0,
|
|
"completions/max_terminated_length": 338.0,
|
|
"completions/mean_length": 235.3125,
|
|
"completions/mean_terminated_length": 235.3125,
|
|
"completions/min_length": 134.2,
|
|
"completions/min_terminated_length": 134.2,
|
|
"entropy": 1.0732504196465016,
|
|
"epoch": 0.4817504826698538,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.815831661224365,
|
|
"kl": 0.038973855052608995,
|
|
"learning_rate": 4.834817084044994e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 22703733.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.4262101650238037,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.42621017098426817,
|
|
"sampling/importance_sampling_ratio/max": 1.3021227717399597,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006640374660491,
|
|
"sampling/importance_sampling_ratio/min": 0.7851735472679138,
|
|
"sampling/sampling_logp_difference/max": 0.2739617943763733,
|
|
"sampling/sampling_logp_difference/mean": 0.02052662502974272,
|
|
"step": 5240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.0,
|
|
"completions/max_terminated_length": 314.0,
|
|
"completions/mean_length": 204.6,
|
|
"completions/mean_terminated_length": 204.6,
|
|
"completions/min_length": 128.3,
|
|
"completions/min_terminated_length": 128.3,
|
|
"entropy": 1.0754360709339381,
|
|
"epoch": 0.4826698538199871,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.600217580795288,
|
|
"kl": 0.030810918210772798,
|
|
"learning_rate": 4.833910205179096e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 22745581.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.40830574929714203,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.40830577313899996,
|
|
"sampling/importance_sampling_ratio/max": 1.280623996257782,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000560104846954,
|
|
"sampling/importance_sampling_ratio/min": 0.7976553618907929,
|
|
"sampling/sampling_logp_difference/max": 0.26327252984046934,
|
|
"sampling/sampling_logp_difference/mean": 0.02123839780688286,
|
|
"step": 5250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.2,
|
|
"completions/max_terminated_length": 314.2,
|
|
"completions/mean_length": 215.4875,
|
|
"completions/mean_terminated_length": 215.4875,
|
|
"completions/min_length": 145.7,
|
|
"completions/min_terminated_length": 145.7,
|
|
"entropy": 1.126778892427683,
|
|
"epoch": 0.4835892249701204,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.7373857498168945,
|
|
"kl": 0.04488398185931146,
|
|
"learning_rate": 4.833000929200331e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 22787508.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.518820071220398,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.5188200891017913,
|
|
"sampling/importance_sampling_ratio/max": 1.2564231872558593,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996687948703766,
|
|
"sampling/importance_sampling_ratio/min": 0.7922789454460144,
|
|
"sampling/sampling_logp_difference/max": 0.25233088731765746,
|
|
"sampling/sampling_logp_difference/mean": 0.021650151722133158,
|
|
"step": 5260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.7,
|
|
"completions/max_terminated_length": 313.7,
|
|
"completions/mean_length": 232.7,
|
|
"completions/mean_terminated_length": 232.7,
|
|
"completions/min_length": 163.0,
|
|
"completions/min_terminated_length": 163.0,
|
|
"entropy": 1.0743748657405376,
|
|
"epoch": 0.48450859612025376,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.6547369956970215,
|
|
"kl": 0.045266679662745446,
|
|
"learning_rate": 4.8320892570426e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 22832148.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.5091026455163956,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.509102675318718,
|
|
"sampling/importance_sampling_ratio/max": 1.3167165398597718,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006289899349212,
|
|
"sampling/importance_sampling_ratio/min": 0.7421549737453461,
|
|
"sampling/sampling_logp_difference/max": 0.3439781904220581,
|
|
"sampling/sampling_logp_difference/mean": 0.02123083993792534,
|
|
"step": 5270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.7,
|
|
"completions/max_terminated_length": 271.7,
|
|
"completions/mean_length": 183.125,
|
|
"completions/mean_terminated_length": 183.125,
|
|
"completions/min_length": 117.4,
|
|
"completions/min_terminated_length": 117.4,
|
|
"entropy": 1.0387254439294338,
|
|
"epoch": 0.48542796727038706,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 9.258883476257324,
|
|
"kl": 0.04214065321721137,
|
|
"learning_rate": 4.831175189642261e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 22869646.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.3799549013376236,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.37995490729808806,
|
|
"sampling/importance_sampling_ratio/max": 1.3570650696754456,
|
|
"sampling/importance_sampling_ratio/mean": 1.000565952062607,
|
|
"sampling/importance_sampling_ratio/min": 0.7757083117961884,
|
|
"sampling/sampling_logp_difference/max": 0.3312030553817749,
|
|
"sampling/sampling_logp_difference/mean": 0.021324425749480725,
|
|
"step": 5280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.4,
|
|
"completions/max_terminated_length": 280.4,
|
|
"completions/mean_length": 195.0,
|
|
"completions/mean_terminated_length": 195.0,
|
|
"completions/min_length": 126.3,
|
|
"completions/min_terminated_length": 126.3,
|
|
"entropy": 1.1406998798251151,
|
|
"epoch": 0.48634733842052036,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14811979234218597,
|
|
"kl": 0.045686901442240926,
|
|
"learning_rate": 4.8302587279381365e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 22908750.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.4017985433340073,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.40179855525493624,
|
|
"sampling/importance_sampling_ratio/max": 1.266722023487091,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002274930477142,
|
|
"sampling/importance_sampling_ratio/min": 0.7796203076839447,
|
|
"sampling/sampling_logp_difference/max": 0.2722718119621277,
|
|
"sampling/sampling_logp_difference/mean": 0.022155637852847576,
|
|
"step": 5290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.3,
|
|
"completions/max_terminated_length": 295.3,
|
|
"completions/mean_length": 209.7875,
|
|
"completions/mean_terminated_length": 209.7875,
|
|
"completions/min_length": 135.1,
|
|
"completions/min_terminated_length": 135.1,
|
|
"entropy": 1.0300621554255485,
|
|
"epoch": 0.48726670957065366,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1667054146528244,
|
|
"kl": 0.04274160105269402,
|
|
"learning_rate": 4.829339872871505e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 22951013.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.29780173897743223,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.29780174493789674,
|
|
"sampling/importance_sampling_ratio/max": 1.250805640220642,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995734453201294,
|
|
"sampling/importance_sampling_ratio/min": 0.7733888864517212,
|
|
"sampling/sampling_logp_difference/max": 0.27021238803863523,
|
|
"sampling/sampling_logp_difference/mean": 0.020219090208411217,
|
|
"step": 5300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 326.9,
|
|
"completions/max_terminated_length": 314.2,
|
|
"completions/mean_length": 217.8375,
|
|
"completions/mean_terminated_length": 215.01785888671876,
|
|
"completions/min_length": 140.6,
|
|
"completions/min_terminated_length": 140.6,
|
|
"entropy": 1.0944420523941516,
|
|
"epoch": 0.488186080720787,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.961233615875244,
|
|
"kl": 0.05282329951878637,
|
|
"learning_rate": 4.828418625386105e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 22993328.0,
|
|
"reward": -0.06875,
|
|
"reward_std": 0.49629092812538145,
|
|
"rewards/reward_fn/mean": -0.06875,
|
|
"rewards/reward_fn/std": 0.4962909460067749,
|
|
"sampling/importance_sampling_ratio/max": 1.2744558334350586,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995882093906403,
|
|
"sampling/importance_sampling_ratio/min": 0.7371620237827301,
|
|
"sampling/sampling_logp_difference/max": 0.3139346122741699,
|
|
"sampling/sampling_logp_difference/mean": 0.02146372217684984,
|
|
"step": 5310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.0,
|
|
"completions/max_terminated_length": 303.0,
|
|
"completions/mean_length": 195.6,
|
|
"completions/mean_terminated_length": 195.6,
|
|
"completions/min_length": 118.2,
|
|
"completions/min_terminated_length": 118.2,
|
|
"entropy": 1.0200982823967935,
|
|
"epoch": 0.4891054518709203,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.7303314208984375,
|
|
"kl": 0.03553521225694567,
|
|
"learning_rate": 4.8274949864281304e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 23034368.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.3961169719696045,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.39611698389053346,
|
|
"sampling/importance_sampling_ratio/max": 1.269530713558197,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001214146614075,
|
|
"sampling/importance_sampling_ratio/min": 0.7562114894390106,
|
|
"sampling/sampling_logp_difference/max": 0.3183754563331604,
|
|
"sampling/sampling_logp_difference/mean": 0.0205043263733387,
|
|
"step": 5320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.3,
|
|
"completions/max_terminated_length": 295.3,
|
|
"completions/mean_length": 208.6125,
|
|
"completions/mean_terminated_length": 208.6125,
|
|
"completions/min_length": 144.2,
|
|
"completions/min_terminated_length": 144.2,
|
|
"entropy": 1.0772612024098636,
|
|
"epoch": 0.4900248230210536,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.967926979064941,
|
|
"kl": 0.04195050619309768,
|
|
"learning_rate": 4.826568956946233e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 23077857.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3446744382381439,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.3446744501590729,
|
|
"sampling/importance_sampling_ratio/max": 1.2724981427192688,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001750588417053,
|
|
"sampling/importance_sampling_ratio/min": 0.794041758775711,
|
|
"sampling/sampling_logp_difference/max": 0.2588728666305542,
|
|
"sampling/sampling_logp_difference/mean": 0.021231592260301115,
|
|
"step": 5330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.7,
|
|
"completions/max_terminated_length": 284.7,
|
|
"completions/mean_length": 209.325,
|
|
"completions/mean_terminated_length": 209.325,
|
|
"completions/min_length": 142.4,
|
|
"completions/min_terminated_length": 142.4,
|
|
"entropy": 1.0945632435381412,
|
|
"epoch": 0.4909441941711869,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 9.656747817993164,
|
|
"kl": 0.04670930468128063,
|
|
"learning_rate": 4.825640537891521e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 23121091.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.44562808573246004,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.4456281214952469,
|
|
"sampling/importance_sampling_ratio/max": 1.348485779762268,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002808809280395,
|
|
"sampling/importance_sampling_ratio/min": 0.7791234493255615,
|
|
"sampling/sampling_logp_difference/max": 0.3120794057846069,
|
|
"sampling/sampling_logp_difference/mean": 0.022376133129000664,
|
|
"step": 5340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.5,
|
|
"completions/max_terminated_length": 284.5,
|
|
"completions/mean_length": 192.9625,
|
|
"completions/mean_terminated_length": 192.9625,
|
|
"completions/min_length": 128.1,
|
|
"completions/min_terminated_length": 128.1,
|
|
"entropy": 1.1277014277875423,
|
|
"epoch": 0.4918635653213202,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.8403520584106445,
|
|
"kl": 0.030757042882032692,
|
|
"learning_rate": 4.824709730217553e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 23161264.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.2833965331315994,
|
|
"rewards/reward_fn/mean": 0.3875,
|
|
"rewards/reward_fn/std": 0.28339654505252837,
|
|
"sampling/importance_sampling_ratio/max": 1.319850754737854,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002212703227997,
|
|
"sampling/importance_sampling_ratio/min": 0.7982283234596252,
|
|
"sampling/sampling_logp_difference/max": 0.28737926483154297,
|
|
"sampling/sampling_logp_difference/mean": 0.022109304368495942,
|
|
"step": 5350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.7,
|
|
"completions/max_terminated_length": 316.7,
|
|
"completions/mean_length": 212.0625,
|
|
"completions/mean_terminated_length": 212.0625,
|
|
"completions/min_length": 130.4,
|
|
"completions/min_terminated_length": 130.4,
|
|
"entropy": 1.0923405088484288,
|
|
"epoch": 0.49278293647145355,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.041152477264404,
|
|
"kl": 0.054811205936130133,
|
|
"learning_rate": 4.8237765348803435e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 23208213.0,
|
|
"reward": -0.275,
|
|
"reward_std": 0.3915108531713486,
|
|
"rewards/reward_fn/mean": -0.275,
|
|
"rewards/reward_fn/std": 0.3915108531713486,
|
|
"sampling/importance_sampling_ratio/max": 1.2654842853546142,
|
|
"sampling/importance_sampling_ratio/mean": 0.999446302652359,
|
|
"sampling/importance_sampling_ratio/min": 0.7852972567081451,
|
|
"sampling/sampling_logp_difference/max": 0.26122102737426756,
|
|
"sampling/sampling_logp_difference/mean": 0.02175639197230339,
|
|
"step": 5360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.5,
|
|
"completions/max_terminated_length": 301.5,
|
|
"completions/mean_length": 202.9875,
|
|
"completions/mean_terminated_length": 202.9875,
|
|
"completions/min_length": 115.8,
|
|
"completions/min_terminated_length": 115.8,
|
|
"entropy": 1.031253406777978,
|
|
"epoch": 0.49370230762158684,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.265824317932129,
|
|
"kl": 0.03812641363474541,
|
|
"learning_rate": 4.8228409528383595e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 23250668.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.2920606702566147,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.29206068217754366,
|
|
"sampling/importance_sampling_ratio/max": 1.2718768835067749,
|
|
"sampling/importance_sampling_ratio/mean": 0.999932837486267,
|
|
"sampling/importance_sampling_ratio/min": 0.8028461933135986,
|
|
"sampling/sampling_logp_difference/max": 0.25324827432632446,
|
|
"sampling/sampling_logp_difference/mean": 0.02090995218604803,
|
|
"step": 5370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.5,
|
|
"completions/max_terminated_length": 289.5,
|
|
"completions/mean_length": 207.475,
|
|
"completions/mean_terminated_length": 207.475,
|
|
"completions/min_length": 140.3,
|
|
"completions/min_terminated_length": 140.3,
|
|
"entropy": 1.0277273558080195,
|
|
"epoch": 0.49462167877172014,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.609491348266602,
|
|
"kl": 0.05440606109332293,
|
|
"learning_rate": 4.821902985052518e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 23292042.0,
|
|
"reward": 0.24375,
|
|
"reward_std": 0.2875739336013794,
|
|
"rewards/reward_fn/mean": 0.24375,
|
|
"rewards/reward_fn/std": 0.2875739336013794,
|
|
"sampling/importance_sampling_ratio/max": 1.2699689745903016,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999425411224365,
|
|
"sampling/importance_sampling_ratio/min": 0.791676652431488,
|
|
"sampling/sampling_logp_difference/max": 0.25165589451789855,
|
|
"sampling/sampling_logp_difference/mean": 0.021200604364275934,
|
|
"step": 5380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.6,
|
|
"completions/max_terminated_length": 325.6,
|
|
"completions/mean_length": 231.3125,
|
|
"completions/mean_terminated_length": 231.3125,
|
|
"completions/min_length": 139.5,
|
|
"completions/min_terminated_length": 139.5,
|
|
"entropy": 1.0847119815647601,
|
|
"epoch": 0.49554104992185344,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.297759532928467,
|
|
"kl": 0.034514278906863184,
|
|
"learning_rate": 4.820962632486187e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 23337795.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.40591218173503874,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.4059121936559677,
|
|
"sampling/importance_sampling_ratio/max": 1.3074236512184143,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004106163978577,
|
|
"sampling/importance_sampling_ratio/min": 0.724758043885231,
|
|
"sampling/sampling_logp_difference/max": 0.354433536529541,
|
|
"sampling/sampling_logp_difference/mean": 0.021418475918471812,
|
|
"step": 5390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.1,
|
|
"completions/max_terminated_length": 298.1,
|
|
"completions/mean_length": 216.05,
|
|
"completions/mean_terminated_length": 216.05,
|
|
"completions/min_length": 150.3,
|
|
"completions/min_terminated_length": 150.3,
|
|
"entropy": 1.0616689071059227,
|
|
"epoch": 0.49646042107198673,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.7572455406188965,
|
|
"kl": 0.036152307875454426,
|
|
"learning_rate": 4.820019896105186e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 23381615.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.43865286409854887,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.4386528700590134,
|
|
"sampling/importance_sampling_ratio/max": 1.2574617743492127,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003314852714538,
|
|
"sampling/importance_sampling_ratio/min": 0.7687016904354096,
|
|
"sampling/sampling_logp_difference/max": 0.28773764371871946,
|
|
"sampling/sampling_logp_difference/mean": 0.021228195913136006,
|
|
"step": 5400
|
|
},
|
|
{
|
|
"epoch": 0.49646042107198673,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0028359511343804536,
|
|
"eval_completions/max_length": 307.5602094240838,
|
|
"eval_completions/max_terminated_length": 305.1378708551483,
|
|
"eval_completions/mean_length": 214.98887434554973,
|
|
"eval_completions/mean_terminated_length": 214.2825357593583,
|
|
"eval_completions/min_length": 138.60383944153577,
|
|
"eval_completions/min_terminated_length": 138.60383944153577,
|
|
"eval_entropy": 1.097362355516011,
|
|
"eval_frac_reward_zero_std": 0.2705061082024433,
|
|
"eval_kl": 0.041150790996797526,
|
|
"eval_loss": 0.002088259207084775,
|
|
"eval_num_tokens": 23381615.0,
|
|
"eval_reward": 0.04602966841186736,
|
|
"eval_reward_std": 0.38701856157126435,
|
|
"eval_rewards/reward_fn/mean": 0.04602966841186736,
|
|
"eval_rewards/reward_fn/std": 0.38701857280564766,
|
|
"eval_runtime": 1785.6175,
|
|
"eval_samples_per_second": 0.321,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2632229927322627,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999714185846206,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7768413987251268,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27680796405615815,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020709345676329954,
|
|
"eval_steps_per_second": 0.04,
|
|
"step": 5400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.2,
|
|
"completions/max_terminated_length": 282.2,
|
|
"completions/mean_length": 208.7125,
|
|
"completions/mean_terminated_length": 208.7125,
|
|
"completions/min_length": 157.2,
|
|
"completions/min_terminated_length": 157.2,
|
|
"entropy": 1.0855384014546872,
|
|
"epoch": 0.4973797922221201,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.11374184489250183,
|
|
"kl": 0.04197303925757297,
|
|
"learning_rate": 4.819074776877778e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 23430008.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.345512917637825,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.34551292955875396,
|
|
"sampling/importance_sampling_ratio/max": 1.295212173461914,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001162230968474,
|
|
"sampling/importance_sampling_ratio/min": 0.7823019981384277,
|
|
"sampling/sampling_logp_difference/max": 0.29045695066452026,
|
|
"sampling/sampling_logp_difference/mean": 0.021257317438721657,
|
|
"step": 5410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 331.3,
|
|
"completions/max_terminated_length": 326.7,
|
|
"completions/mean_length": 225.75,
|
|
"completions/mean_terminated_length": 222.89642944335938,
|
|
"completions/min_length": 149.7,
|
|
"completions/min_terminated_length": 149.7,
|
|
"entropy": 1.088605421036482,
|
|
"epoch": 0.4982991633722534,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.4198384284973145,
|
|
"kl": 0.05059795609558933,
|
|
"learning_rate": 4.818127275774678e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 23475036.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.5018741875886917,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.5018741935491562,
|
|
"sampling/importance_sampling_ratio/max": 1.2814471125602722,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002063870429994,
|
|
"sampling/importance_sampling_ratio/min": 0.771407401561737,
|
|
"sampling/sampling_logp_difference/max": 0.27978352904319764,
|
|
"sampling/sampling_logp_difference/mean": 0.02201297674328089,
|
|
"step": 5420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.7,
|
|
"completions/max_terminated_length": 309.7,
|
|
"completions/mean_length": 220.2875,
|
|
"completions/mean_terminated_length": 220.2875,
|
|
"completions/min_length": 144.3,
|
|
"completions/min_terminated_length": 144.3,
|
|
"entropy": 1.1716218397021294,
|
|
"epoch": 0.4992185345223867,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.18368206918239594,
|
|
"kl": 0.046855731157120314,
|
|
"learning_rate": 4.817177393769046e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 23522891.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.4315427541732788,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.43154277205467223,
|
|
"sampling/importance_sampling_ratio/max": 1.2900241613388062,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004517495632173,
|
|
"sampling/importance_sampling_ratio/min": 0.7981406569480896,
|
|
"sampling/sampling_logp_difference/max": 0.26332831382751465,
|
|
"sampling/sampling_logp_difference/mean": 0.022118272632360457,
|
|
"step": 5430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.1,
|
|
"completions/max_terminated_length": 313.1,
|
|
"completions/mean_length": 208.7125,
|
|
"completions/mean_terminated_length": 208.7125,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.0546998620033263,
|
|
"epoch": 0.50013790567252,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 4.902407646179199,
|
|
"kl": 0.04262739559053443,
|
|
"learning_rate": 4.816225131836485e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 23566548.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.20353024303913117,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.20353024303913117,
|
|
"sampling/importance_sampling_ratio/max": 1.2638328313827514,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998726129531861,
|
|
"sampling/importance_sampling_ratio/min": 0.8027492523193359,
|
|
"sampling/sampling_logp_difference/max": 0.2428574800491333,
|
|
"sampling/sampling_logp_difference/mean": 0.021070289239287377,
|
|
"step": 5440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.7,
|
|
"completions/max_terminated_length": 275.7,
|
|
"completions/mean_length": 197.15,
|
|
"completions/mean_terminated_length": 197.15,
|
|
"completions/min_length": 133.4,
|
|
"completions/min_terminated_length": 133.4,
|
|
"entropy": 1.0130454123020172,
|
|
"epoch": 0.5010572768226533,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.8741374015808105,
|
|
"kl": 0.03473435846390203,
|
|
"learning_rate": 4.8152704909550455e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 23609680.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.2958350956439972,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.29583510756492615,
|
|
"sampling/importance_sampling_ratio/max": 1.2904534578323363,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997602343559265,
|
|
"sampling/importance_sampling_ratio/min": 0.7649597525596619,
|
|
"sampling/sampling_logp_difference/max": 0.31208584308624265,
|
|
"sampling/sampling_logp_difference/mean": 0.020262810960412027,
|
|
"step": 5450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.7,
|
|
"completions/max_terminated_length": 325.7,
|
|
"completions/mean_length": 213.925,
|
|
"completions/mean_terminated_length": 213.925,
|
|
"completions/min_length": 146.2,
|
|
"completions/min_terminated_length": 146.2,
|
|
"entropy": 1.1191648043692113,
|
|
"epoch": 0.5019766479727866,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.293253898620605,
|
|
"kl": 0.053668214788194744,
|
|
"learning_rate": 4.8143134721052205e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 23655090.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.3743988037109375,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.37439881563186644,
|
|
"sampling/importance_sampling_ratio/max": 1.277535653114319,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996655941009521,
|
|
"sampling/importance_sampling_ratio/min": 0.7746275663375854,
|
|
"sampling/sampling_logp_difference/max": 0.2920875698328018,
|
|
"sampling/sampling_logp_difference/mean": 0.021859637834131716,
|
|
"step": 5460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.8,
|
|
"completions/max_terminated_length": 282.8,
|
|
"completions/mean_length": 211.0625,
|
|
"completions/mean_terminated_length": 211.0625,
|
|
"completions/min_length": 142.2,
|
|
"completions/min_terminated_length": 142.2,
|
|
"entropy": 1.0439486525952817,
|
|
"epoch": 0.5028960191229199,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.73133659362793,
|
|
"kl": 0.04664284294121899,
|
|
"learning_rate": 4.8133540762699445e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 23699231.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.279134601354599,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.27913460731506345,
|
|
"sampling/importance_sampling_ratio/max": 1.2586267232894897,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003837823867798,
|
|
"sampling/importance_sampling_ratio/min": 0.7755716264247894,
|
|
"sampling/sampling_logp_difference/max": 0.26464965343475344,
|
|
"sampling/sampling_logp_difference/mean": 0.021272764541208744,
|
|
"step": 5470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.0,
|
|
"completions/max_terminated_length": 310.0,
|
|
"completions/mean_length": 214.0625,
|
|
"completions/mean_terminated_length": 214.0625,
|
|
"completions/min_length": 126.8,
|
|
"completions/min_terminated_length": 126.8,
|
|
"entropy": 1.0752910524606705,
|
|
"epoch": 0.5038153902730532,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.440293788909912,
|
|
"kl": 0.04500622688210569,
|
|
"learning_rate": 4.812392304434595e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 23741532.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.30942772030830384,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.30942773818969727,
|
|
"sampling/importance_sampling_ratio/max": 1.320425820350647,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999340176582336,
|
|
"sampling/importance_sampling_ratio/min": 0.7474205732345581,
|
|
"sampling/sampling_logp_difference/max": 0.3206120550632477,
|
|
"sampling/sampling_logp_difference/mean": 0.022241763584315778,
|
|
"step": 5480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.7,
|
|
"completions/max_terminated_length": 280.7,
|
|
"completions/mean_length": 185.2875,
|
|
"completions/mean_terminated_length": 185.2875,
|
|
"completions/min_length": 116.8,
|
|
"completions/min_terminated_length": 116.8,
|
|
"entropy": 1.1046483531594276,
|
|
"epoch": 0.5047347614231865,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.75298547744751,
|
|
"kl": 0.0329540359089151,
|
|
"learning_rate": 4.811428157586987e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 23781219.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.25470708310604095,
|
|
"rewards/reward_fn/mean": 0.4375,
|
|
"rewards/reward_fn/std": 0.2547071009874344,
|
|
"sampling/importance_sampling_ratio/max": 1.2729754805564881,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001352250576019,
|
|
"sampling/importance_sampling_ratio/min": 0.7876677334308624,
|
|
"sampling/sampling_logp_difference/max": 0.2548651576042175,
|
|
"sampling/sampling_logp_difference/mean": 0.022425617650151254,
|
|
"step": 5490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.8,
|
|
"completions/max_terminated_length": 305.8,
|
|
"completions/mean_length": 219.45,
|
|
"completions/mean_terminated_length": 219.45,
|
|
"completions/min_length": 145.1,
|
|
"completions/min_terminated_length": 145.1,
|
|
"entropy": 1.0868753015995025,
|
|
"epoch": 0.5056541325733198,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.9453043937683105,
|
|
"kl": 0.035435753501951696,
|
|
"learning_rate": 4.810461636717378e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 23828383.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.1974882572889328,
|
|
"rewards/reward_fn/mean": 0.3625,
|
|
"rewards/reward_fn/std": 0.19748826324939728,
|
|
"sampling/importance_sampling_ratio/max": 1.2394713640213013,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994851529598237,
|
|
"sampling/importance_sampling_ratio/min": 0.7812922775745392,
|
|
"sampling/sampling_logp_difference/max": 0.27424551248550416,
|
|
"sampling/sampling_logp_difference/mean": 0.02108355090022087,
|
|
"step": 5500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 340.6,
|
|
"completions/max_terminated_length": 340.6,
|
|
"completions/mean_length": 240.4,
|
|
"completions/mean_terminated_length": 240.4,
|
|
"completions/min_length": 161.1,
|
|
"completions/min_terminated_length": 161.1,
|
|
"entropy": 1.080581258237362,
|
|
"epoch": 0.5065735037234531,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.3365761637687683,
|
|
"kl": 0.04969004578888416,
|
|
"learning_rate": 4.809492742818461e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 23873775.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.3202547788619995,
|
|
"sampling/importance_sampling_ratio/max": 1.3583197593688965,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000873923301696,
|
|
"sampling/importance_sampling_ratio/min": 0.7611360907554626,
|
|
"sampling/sampling_logp_difference/max": 0.36482442617416383,
|
|
"sampling/sampling_logp_difference/mean": 0.02113449703902006,
|
|
"step": 5510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.7,
|
|
"completions/max_terminated_length": 312.7,
|
|
"completions/mean_length": 235.7375,
|
|
"completions/mean_terminated_length": 235.7375,
|
|
"completions/min_length": 148.3,
|
|
"completions/min_terminated_length": 148.3,
|
|
"entropy": 1.1689024038612843,
|
|
"epoch": 0.5074928748735865,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.155808225274086,
|
|
"kl": 0.0653985949466005,
|
|
"learning_rate": 4.808521476885369e-06,
|
|
"loss": 0.0035,
|
|
"num_tokens": 23920354.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.33921052813529967,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.3392105340957642,
|
|
"sampling/importance_sampling_ratio/max": 1.2750370264053346,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001432359218598,
|
|
"sampling/importance_sampling_ratio/min": 0.7733922660350799,
|
|
"sampling/sampling_logp_difference/max": 0.2868382573127747,
|
|
"sampling/sampling_logp_difference/mean": 0.022046180628240108,
|
|
"step": 5520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 345.0,
|
|
"completions/max_terminated_length": 345.0,
|
|
"completions/mean_length": 244.65,
|
|
"completions/mean_terminated_length": 244.65,
|
|
"completions/min_length": 154.9,
|
|
"completions/min_terminated_length": 154.9,
|
|
"entropy": 1.118749164789915,
|
|
"epoch": 0.5084122460237198,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.13883113861084,
|
|
"kl": 0.04233728646649979,
|
|
"learning_rate": 4.80754783991567e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 23968942.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.3486855089664459,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.34868552088737487,
|
|
"sampling/importance_sampling_ratio/max": 1.2819775581359862,
|
|
"sampling/importance_sampling_ratio/mean": 0.9992317497730255,
|
|
"sampling/importance_sampling_ratio/min": 0.7228488445281982,
|
|
"sampling/sampling_logp_difference/max": 0.35451061725616456,
|
|
"sampling/sampling_logp_difference/mean": 0.020751421339809895,
|
|
"step": 5530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.7,
|
|
"completions/max_terminated_length": 281.7,
|
|
"completions/mean_length": 203.075,
|
|
"completions/mean_terminated_length": 203.075,
|
|
"completions/min_length": 123.3,
|
|
"completions/min_terminated_length": 123.3,
|
|
"entropy": 1.0664518512785435,
|
|
"epoch": 0.5093316171738531,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.4434919357299805,
|
|
"kl": 0.03897197861224413,
|
|
"learning_rate": 4.806571832909367e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 24013916.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.2411646068096161,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.2411646068096161,
|
|
"sampling/importance_sampling_ratio/max": 1.2860379338264465,
|
|
"sampling/importance_sampling_ratio/mean": 0.9992936015129089,
|
|
"sampling/importance_sampling_ratio/min": 0.7554073631763458,
|
|
"sampling/sampling_logp_difference/max": 0.31667001247406007,
|
|
"sampling/sampling_logp_difference/mean": 0.021709810197353362,
|
|
"step": 5540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.6,
|
|
"completions/max_terminated_length": 331.6,
|
|
"completions/mean_length": 210.875,
|
|
"completions/mean_terminated_length": 210.875,
|
|
"completions/min_length": 120.8,
|
|
"completions/min_terminated_length": 120.8,
|
|
"entropy": 1.0497647486627102,
|
|
"epoch": 0.5102509883239864,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 7.073035717010498,
|
|
"kl": 0.05006906929193065,
|
|
"learning_rate": 4.805593456868897e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 24056306.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.23831535279750823,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.23831535279750823,
|
|
"sampling/importance_sampling_ratio/max": 1.2860930919647218,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004346251487732,
|
|
"sampling/importance_sampling_ratio/min": 0.7597083330154419,
|
|
"sampling/sampling_logp_difference/max": 0.3008319616317749,
|
|
"sampling/sampling_logp_difference/mean": 0.021445170044898987,
|
|
"step": 5550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.7,
|
|
"completions/max_terminated_length": 299.7,
|
|
"completions/mean_length": 201.975,
|
|
"completions/mean_terminated_length": 201.975,
|
|
"completions/min_length": 131.6,
|
|
"completions/min_terminated_length": 131.6,
|
|
"entropy": 1.026543417572975,
|
|
"epoch": 0.5111703594741197,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.548365116119385,
|
|
"kl": 0.03880849097040482,
|
|
"learning_rate": 4.804612712799131e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 24096016.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.45168755352497103,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.45168757140636445,
|
|
"sampling/importance_sampling_ratio/max": 1.2572989583015441,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003212630748748,
|
|
"sampling/importance_sampling_ratio/min": 0.7860771536827087,
|
|
"sampling/sampling_logp_difference/max": 0.25114492177963255,
|
|
"sampling/sampling_logp_difference/mean": 0.020918383449316024,
|
|
"step": 5560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.0,
|
|
"completions/max_terminated_length": 287.0,
|
|
"completions/mean_length": 200.575,
|
|
"completions/mean_terminated_length": 200.575,
|
|
"completions/min_length": 115.9,
|
|
"completions/min_terminated_length": 115.9,
|
|
"entropy": 1.017763302475214,
|
|
"epoch": 0.512089730624253,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.291685581207275,
|
|
"kl": 0.039697503374191,
|
|
"learning_rate": 4.803629601707371e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 24137646.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.36288144290447233,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.36288144886493684,
|
|
"sampling/importance_sampling_ratio/max": 1.258324921131134,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002608120441436,
|
|
"sampling/importance_sampling_ratio/min": 0.7740411341190339,
|
|
"sampling/sampling_logp_difference/max": 0.26839998066425325,
|
|
"sampling/sampling_logp_difference/mean": 0.020473778247833252,
|
|
"step": 5570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 263.5,
|
|
"completions/max_terminated_length": 263.5,
|
|
"completions/mean_length": 194.0125,
|
|
"completions/mean_terminated_length": 194.0125,
|
|
"completions/min_length": 115.3,
|
|
"completions/min_terminated_length": 115.3,
|
|
"entropy": 1.1239424429833889,
|
|
"epoch": 0.5130091017743863,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 4.650895595550537,
|
|
"kl": 0.04226044338429347,
|
|
"learning_rate": 4.802644124603352e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 24179487.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.2597990781068802,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.2597990781068802,
|
|
"sampling/importance_sampling_ratio/max": 1.2658164501190186,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998365223407746,
|
|
"sampling/importance_sampling_ratio/min": 0.743103152513504,
|
|
"sampling/sampling_logp_difference/max": 0.32296501398086547,
|
|
"sampling/sampling_logp_difference/mean": 0.022668720223009586,
|
|
"step": 5580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 327.4,
|
|
"completions/max_terminated_length": 321.8,
|
|
"completions/mean_length": 226.8875,
|
|
"completions/mean_terminated_length": 223.66071472167968,
|
|
"completions/min_length": 146.3,
|
|
"completions/min_terminated_length": 146.3,
|
|
"entropy": 1.0531333975493908,
|
|
"epoch": 0.5139284729245196,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.46544885635376,
|
|
"kl": 0.06050649849930778,
|
|
"learning_rate": 4.8016562824992375e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 24222814.0,
|
|
"reward": -0.24375,
|
|
"reward_std": 0.32643375992774964,
|
|
"rewards/reward_fn/mean": -0.24375,
|
|
"rewards/reward_fn/std": 0.32643375992774964,
|
|
"sampling/importance_sampling_ratio/max": 1.4233837723731995,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004831790924071,
|
|
"sampling/importance_sampling_ratio/min": 0.7718040168285369,
|
|
"sampling/sampling_logp_difference/max": 0.37423638105392454,
|
|
"sampling/sampling_logp_difference/mean": 0.020690672285854816,
|
|
"step": 5590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.9,
|
|
"completions/max_terminated_length": 324.9,
|
|
"completions/mean_length": 241.8,
|
|
"completions/mean_terminated_length": 241.8,
|
|
"completions/min_length": 167.0,
|
|
"completions/min_terminated_length": 167.0,
|
|
"entropy": 1.044118883088231,
|
|
"epoch": 0.5148478440746529,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.0964155197143555,
|
|
"kl": 0.04273007106385194,
|
|
"learning_rate": 4.80066607640962e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 24269926.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.28198385536670684,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.28198386132717135,
|
|
"sampling/importance_sampling_ratio/max": 1.2610069870948792,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994114875793457,
|
|
"sampling/importance_sampling_ratio/min": 0.7865147888660431,
|
|
"sampling/sampling_logp_difference/max": 0.2615241646766663,
|
|
"sampling/sampling_logp_difference/mean": 0.020354469120502473,
|
|
"step": 5600
|
|
},
|
|
{
|
|
"epoch": 0.5148478440746529,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.005671902268760907,
|
|
"eval_completions/max_length": 315.58987783595114,
|
|
"eval_completions/max_terminated_length": 309.56893542757416,
|
|
"eval_completions/mean_length": 219.44328097731238,
|
|
"eval_completions/mean_terminated_length": 217.974965067226,
|
|
"eval_completions/min_length": 140.5130890052356,
|
|
"eval_completions/min_terminated_length": 140.5130890052356,
|
|
"eval_entropy": 1.1105714183931368,
|
|
"eval_frac_reward_zero_std": 0.30715532286212915,
|
|
"eval_kl": 0.04778910652908251,
|
|
"eval_loss": 0.0024661123752593994,
|
|
"eval_num_tokens": 24269926.0,
|
|
"eval_reward": 0.0549738219895288,
|
|
"eval_reward_std": 0.34697055358953294,
|
|
"eval_rewards/reward_fn/mean": 0.0549738219895288,
|
|
"eval_rewards/reward_fn/std": 0.34697056378369556,
|
|
"eval_runtime": 1827.6072,
|
|
"eval_samples_per_second": 0.314,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2673593607664524,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000326239209734,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7793972575955782,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2750438252044598,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021003692288985427,
|
|
"eval_steps_per_second": 0.039,
|
|
"step": 5600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 337.7,
|
|
"completions/max_terminated_length": 337.2,
|
|
"completions/mean_length": 233.2125,
|
|
"completions/mean_terminated_length": 227.69166870117186,
|
|
"completions/min_length": 123.1,
|
|
"completions/min_terminated_length": 123.1,
|
|
"entropy": 1.0501929476857186,
|
|
"epoch": 0.5157672152247862,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.607937335968018,
|
|
"kl": 0.06056828326545656,
|
|
"learning_rate": 4.799673507351522e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 24312359.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.46867029964923856,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.46867031753063204,
|
|
"sampling/importance_sampling_ratio/max": 1.2412830114364624,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000840306282044,
|
|
"sampling/importance_sampling_ratio/min": 0.770388787984848,
|
|
"sampling/sampling_logp_difference/max": 0.2749655842781067,
|
|
"sampling/sampling_logp_difference/mean": 0.020925150718539953,
|
|
"step": 5610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.2,
|
|
"completions/max_terminated_length": 282.2,
|
|
"completions/mean_length": 200.6875,
|
|
"completions/mean_terminated_length": 200.6875,
|
|
"completions/min_length": 134.2,
|
|
"completions/min_terminated_length": 134.2,
|
|
"entropy": 1.0710347529500723,
|
|
"epoch": 0.5166865863749196,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.6804656982421875,
|
|
"kl": 0.03572373668430373,
|
|
"learning_rate": 4.79867857634439e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 24352510.0,
|
|
"reward": 0.46875,
|
|
"reward_std": 0.44541222155094146,
|
|
"rewards/reward_fn/mean": 0.46875,
|
|
"rewards/reward_fn/std": 0.4454122453927994,
|
|
"sampling/importance_sampling_ratio/max": 1.3391463160514832,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001074016094207,
|
|
"sampling/importance_sampling_ratio/min": 0.756105387210846,
|
|
"sampling/sampling_logp_difference/max": 0.3262542188167572,
|
|
"sampling/sampling_logp_difference/mean": 0.021790395304560662,
|
|
"step": 5620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.9,
|
|
"completions/max_terminated_length": 313.9,
|
|
"completions/mean_length": 220.125,
|
|
"completions/mean_terminated_length": 220.125,
|
|
"completions/min_length": 145.4,
|
|
"completions/min_terminated_length": 145.4,
|
|
"entropy": 1.098129228502512,
|
|
"epoch": 0.5176059575250529,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.304335594177246,
|
|
"kl": 0.05576671363087371,
|
|
"learning_rate": 4.797681284410099e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 24397704.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.34776818454265596,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.34776818454265596,
|
|
"sampling/importance_sampling_ratio/max": 1.2820573925971985,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999447584152221,
|
|
"sampling/importance_sampling_ratio/min": 0.7967436611652374,
|
|
"sampling/sampling_logp_difference/max": 0.2670341968536377,
|
|
"sampling/sampling_logp_difference/mean": 0.02156980838626623,
|
|
"step": 5630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 317.2,
|
|
"completions/max_terminated_length": 317.2,
|
|
"completions/mean_length": 214.6875,
|
|
"completions/mean_terminated_length": 214.6875,
|
|
"completions/min_length": 133.9,
|
|
"completions/min_terminated_length": 133.9,
|
|
"entropy": 1.016185887902975,
|
|
"epoch": 0.5185253286751862,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.14543519914150238,
|
|
"kl": 0.0483966107130982,
|
|
"learning_rate": 4.796681632572946e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 24437607.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.16875659823417663,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.1687566041946411,
|
|
"sampling/importance_sampling_ratio/max": 1.2632392406463624,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994130074977875,
|
|
"sampling/importance_sampling_ratio/min": 0.7837559282779694,
|
|
"sampling/sampling_logp_difference/max": 0.2636220932006836,
|
|
"sampling/sampling_logp_difference/mean": 0.020958153530955315,
|
|
"step": 5640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.1,
|
|
"completions/max_terminated_length": 314.1,
|
|
"completions/mean_length": 224.1625,
|
|
"completions/mean_terminated_length": 224.1625,
|
|
"completions/min_length": 136.8,
|
|
"completions/min_terminated_length": 136.8,
|
|
"entropy": 1.0933317504823208,
|
|
"epoch": 0.5194446998253195,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.972399711608887,
|
|
"kl": 0.05069115615915507,
|
|
"learning_rate": 4.795679621859656e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 24483484.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.32111361920833587,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.3211136251688004,
|
|
"sampling/importance_sampling_ratio/max": 1.2839259147644042,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994974672794342,
|
|
"sampling/importance_sampling_ratio/min": 0.7433544337749481,
|
|
"sampling/sampling_logp_difference/max": 0.3108355164527893,
|
|
"sampling/sampling_logp_difference/mean": 0.02195710614323616,
|
|
"step": 5650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 304.2,
|
|
"completions/max_terminated_length": 298.8,
|
|
"completions/mean_length": 210.6375,
|
|
"completions/mean_terminated_length": 207.17857208251954,
|
|
"completions/min_length": 131.3,
|
|
"completions/min_terminated_length": 131.3,
|
|
"entropy": 1.0530016131699085,
|
|
"epoch": 0.5203640709754528,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.1349094957113266,
|
|
"kl": 0.054761074238922444,
|
|
"learning_rate": 4.794675253299373e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 24527911.0,
|
|
"reward": -0.18125,
|
|
"reward_std": 0.306527179479599,
|
|
"rewards/reward_fn/mean": -0.18125,
|
|
"rewards/reward_fn/std": 0.30652719736099243,
|
|
"sampling/importance_sampling_ratio/max": 1.2715108275413514,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005002439022064,
|
|
"sampling/importance_sampling_ratio/min": 0.7686617136001587,
|
|
"sampling/sampling_logp_difference/max": 0.2917294502258301,
|
|
"sampling/sampling_logp_difference/mean": 0.021296564675867557,
|
|
"step": 5660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 349.6,
|
|
"completions/max_terminated_length": 328.8,
|
|
"completions/mean_length": 239.1,
|
|
"completions/mean_terminated_length": 235.28214416503906,
|
|
"completions/min_length": 161.5,
|
|
"completions/min_terminated_length": 161.5,
|
|
"entropy": 1.0534119665622712,
|
|
"epoch": 0.5212834421255861,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.625903129577637,
|
|
"kl": 0.05668536044540815,
|
|
"learning_rate": 4.793668527923664e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 24576903.0,
|
|
"reward": 0.06875,
|
|
"reward_std": 0.3632109582424164,
|
|
"rewards/reward_fn/mean": 0.06875,
|
|
"rewards/reward_fn/std": 0.36321096420288085,
|
|
"sampling/importance_sampling_ratio/max": 1.3016605615615844,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999427795410156,
|
|
"sampling/importance_sampling_ratio/min": 0.7347596615552903,
|
|
"sampling/sampling_logp_difference/max": 0.35710935592651366,
|
|
"sampling/sampling_logp_difference/mean": 0.020988625288009644,
|
|
"step": 5670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 321.4,
|
|
"completions/max_terminated_length": 321.4,
|
|
"completions/mean_length": 207.8125,
|
|
"completions/mean_terminated_length": 207.8125,
|
|
"completions/min_length": 127.4,
|
|
"completions/min_terminated_length": 127.4,
|
|
"entropy": 1.0410642627626658,
|
|
"epoch": 0.5222028132757194,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.201572984457016,
|
|
"kl": 0.051045022631296885,
|
|
"learning_rate": 4.792659446766517e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 24620832.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.24485630989074708,
|
|
"rewards/reward_fn/mean": 0.3375,
|
|
"rewards/reward_fn/std": 0.24485631585121154,
|
|
"sampling/importance_sampling_ratio/max": 1.2378967881202698,
|
|
"sampling/importance_sampling_ratio/mean": 1.000049066543579,
|
|
"sampling/importance_sampling_ratio/min": 0.7668177425861359,
|
|
"sampling/sampling_logp_difference/max": 0.28151760101318357,
|
|
"sampling/sampling_logp_difference/mean": 0.021273509226739408,
|
|
"step": 5680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 334.2,
|
|
"completions/max_terminated_length": 315.8,
|
|
"completions/mean_length": 218.5875,
|
|
"completions/mean_terminated_length": 215.53214416503906,
|
|
"completions/min_length": 145.5,
|
|
"completions/min_terminated_length": 145.5,
|
|
"entropy": 1.0421565152704715,
|
|
"epoch": 0.5231221844258527,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.11205685138702393,
|
|
"kl": 0.05851724444655702,
|
|
"learning_rate": 4.791648010864338e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 24664423.0,
|
|
"reward": -0.11875,
|
|
"reward_std": 0.2398865133523941,
|
|
"rewards/reward_fn/mean": -0.11875,
|
|
"rewards/reward_fn/std": 0.2398865193128586,
|
|
"sampling/importance_sampling_ratio/max": 1.250590181350708,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005945086479187,
|
|
"sampling/importance_sampling_ratio/min": 0.7430494248867034,
|
|
"sampling/sampling_logp_difference/max": 0.3044642448425293,
|
|
"sampling/sampling_logp_difference/mean": 0.02128320075571537,
|
|
"step": 5690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.7,
|
|
"completions/max_terminated_length": 290.7,
|
|
"completions/mean_length": 225.8625,
|
|
"completions/mean_terminated_length": 225.8625,
|
|
"completions/min_length": 155.4,
|
|
"completions/min_terminated_length": 155.4,
|
|
"entropy": 1.0327788598835468,
|
|
"epoch": 0.524041555575986,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.10939747095108032,
|
|
"kl": 0.05683743947884068,
|
|
"learning_rate": 4.790634221255955e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 24709876.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.3265853762626648,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.3265853762626648,
|
|
"sampling/importance_sampling_ratio/max": 1.2294748663902282,
|
|
"sampling/importance_sampling_ratio/mean": 0.999708741903305,
|
|
"sampling/importance_sampling_ratio/min": 0.7827854335308075,
|
|
"sampling/sampling_logp_difference/max": 0.2474528133869171,
|
|
"sampling/sampling_logp_difference/mean": 0.020984406210482122,
|
|
"step": 5700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.3,
|
|
"completions/max_terminated_length": 274.3,
|
|
"completions/mean_length": 209.15,
|
|
"completions/mean_terminated_length": 209.15,
|
|
"completions/min_length": 138.7,
|
|
"completions/min_terminated_length": 138.7,
|
|
"entropy": 1.1975786916911602,
|
|
"epoch": 0.5249609267261194,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.12670528888702393,
|
|
"kl": 0.06615693124476821,
|
|
"learning_rate": 4.789618078982609e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 24756248.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.41830068826675415,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.41830070614814757,
|
|
"sampling/importance_sampling_ratio/max": 1.2364379525184632,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996397078037262,
|
|
"sampling/importance_sampling_ratio/min": 0.7818032085895539,
|
|
"sampling/sampling_logp_difference/max": 0.2549152851104736,
|
|
"sampling/sampling_logp_difference/mean": 0.02394681181758642,
|
|
"step": 5710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.1,
|
|
"completions/max_terminated_length": 309.1,
|
|
"completions/mean_length": 203.175,
|
|
"completions/mean_terminated_length": 203.175,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 0.975874250382185,
|
|
"epoch": 0.5258802978762527,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.878521919250488,
|
|
"kl": 0.054947742447257045,
|
|
"learning_rate": 4.788599585087961e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 24798830.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.41799977123737336,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.4179997891187668,
|
|
"sampling/importance_sampling_ratio/max": 1.2630855560302734,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002691149711609,
|
|
"sampling/importance_sampling_ratio/min": 0.768866166472435,
|
|
"sampling/sampling_logp_difference/max": 0.3106554627418518,
|
|
"sampling/sampling_logp_difference/mean": 0.0207120256498456,
|
|
"step": 5720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.1,
|
|
"completions/max_terminated_length": 292.1,
|
|
"completions/mean_length": 200.875,
|
|
"completions/mean_terminated_length": 200.875,
|
|
"completions/min_length": 130.5,
|
|
"completions/min_terminated_length": 130.5,
|
|
"entropy": 1.0568383276462554,
|
|
"epoch": 0.526799669026386,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.458049774169922,
|
|
"kl": 0.059383532774518244,
|
|
"learning_rate": 4.787578740618085e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 24844860.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.23594963550567627,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.23594963550567627,
|
|
"sampling/importance_sampling_ratio/max": 1.2591054797172547,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997957229614258,
|
|
"sampling/importance_sampling_ratio/min": 0.7429640352725982,
|
|
"sampling/sampling_logp_difference/max": 0.31181050539016725,
|
|
"sampling/sampling_logp_difference/mean": 0.02219392582774162,
|
|
"step": 5730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 301.9,
|
|
"completions/max_terminated_length": 280.4,
|
|
"completions/mean_length": 206.6375,
|
|
"completions/mean_terminated_length": 202.7875015258789,
|
|
"completions/min_length": 132.1,
|
|
"completions/min_terminated_length": 132.1,
|
|
"entropy": 1.030839866027236,
|
|
"epoch": 0.5277190401765193,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.549622058868408,
|
|
"kl": 0.06440929214004427,
|
|
"learning_rate": 4.786555546621471e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 24889055.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.25552249550819395,
|
|
"rewards/reward_fn/mean": 0.09375,
|
|
"rewards/reward_fn/std": 0.25552249550819395,
|
|
"sampling/importance_sampling_ratio/max": 1.259114420413971,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000966727733611,
|
|
"sampling/importance_sampling_ratio/min": 0.7558643162250519,
|
|
"sampling/sampling_logp_difference/max": 0.2906364440917969,
|
|
"sampling/sampling_logp_difference/mean": 0.021959654986858368,
|
|
"step": 5740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.7,
|
|
"completions/max_terminated_length": 289.7,
|
|
"completions/mean_length": 204.725,
|
|
"completions/mean_terminated_length": 204.725,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"entropy": 1.1053141940385103,
|
|
"epoch": 0.5286384113266526,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.14527277648448944,
|
|
"kl": 0.05932108196429908,
|
|
"learning_rate": 4.78553000414902e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 24934417.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.18165890872478485,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.18165890872478485,
|
|
"sampling/importance_sampling_ratio/max": 1.3177197337150575,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995461583137513,
|
|
"sampling/importance_sampling_ratio/min": 0.7414583563804626,
|
|
"sampling/sampling_logp_difference/max": 0.3368691593408585,
|
|
"sampling/sampling_logp_difference/mean": 0.02223340328782797,
|
|
"step": 5750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 328.6,
|
|
"completions/max_terminated_length": 328.6,
|
|
"completions/mean_length": 211.9625,
|
|
"completions/mean_terminated_length": 211.9625,
|
|
"completions/min_length": 131.5,
|
|
"completions/min_terminated_length": 131.5,
|
|
"entropy": 1.0027941025793552,
|
|
"epoch": 0.5295577824767859,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.11421628296375275,
|
|
"kl": 0.05854715235182084,
|
|
"learning_rate": 4.784502114254048e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 24975278.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.28198385536670684,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.28198386132717135,
|
|
"sampling/importance_sampling_ratio/max": 1.2803743124008178,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997711777687073,
|
|
"sampling/importance_sampling_ratio/min": 0.7688262104988098,
|
|
"sampling/sampling_logp_difference/max": 0.2703778803348541,
|
|
"sampling/sampling_logp_difference/mean": 0.020782227255403997,
|
|
"step": 5760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.2,
|
|
"completions/max_terminated_length": 296.2,
|
|
"completions/mean_length": 211.9125,
|
|
"completions/mean_terminated_length": 211.9125,
|
|
"completions/min_length": 135.7,
|
|
"completions/min_terminated_length": 135.7,
|
|
"entropy": 1.0665974929928779,
|
|
"epoch": 0.5304771536269192,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1431657075881958,
|
|
"kl": 0.05518575538881123,
|
|
"learning_rate": 4.783471877992279e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 25016967.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.447176194190979,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4471762001514435,
|
|
"sampling/importance_sampling_ratio/max": 1.230408501625061,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002275586128235,
|
|
"sampling/importance_sampling_ratio/min": 0.7900798976421356,
|
|
"sampling/sampling_logp_difference/max": 0.23983997106552124,
|
|
"sampling/sampling_logp_difference/mean": 0.0215107848867774,
|
|
"step": 5770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 320.2,
|
|
"completions/max_terminated_length": 295.7,
|
|
"completions/mean_length": 219.3875,
|
|
"completions/mean_terminated_length": 215.79107208251952,
|
|
"completions/min_length": 150.1,
|
|
"completions/min_terminated_length": 150.1,
|
|
"entropy": 1.0803352303802967,
|
|
"epoch": 0.5313965247770525,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.364077091217041,
|
|
"kl": 0.04843412353657186,
|
|
"learning_rate": 4.782439296421849e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 25060574.0,
|
|
"reward": -0.01875,
|
|
"reward_std": 0.4253198474645615,
|
|
"rewards/reward_fn/mean": -0.01875,
|
|
"rewards/reward_fn/std": 0.42531985342502593,
|
|
"sampling/importance_sampling_ratio/max": 1.2883960366249085,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003319561481476,
|
|
"sampling/importance_sampling_ratio/min": 0.7825756907463074,
|
|
"sampling/sampling_logp_difference/max": 0.2829066514968872,
|
|
"sampling/sampling_logp_difference/mean": 0.020909501053392888,
|
|
"step": 5780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.9,
|
|
"completions/max_terminated_length": 285.9,
|
|
"completions/mean_length": 205.125,
|
|
"completions/mean_terminated_length": 205.125,
|
|
"completions/min_length": 138.2,
|
|
"completions/min_terminated_length": 138.2,
|
|
"entropy": 1.0578637324273585,
|
|
"epoch": 0.5323158959271858,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13388663530349731,
|
|
"kl": 0.03832183027989231,
|
|
"learning_rate": 4.781404370603301e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 25103304.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.31589864790439603,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.31589865386486055,
|
|
"sampling/importance_sampling_ratio/max": 1.2577978253364563,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995961129665375,
|
|
"sampling/importance_sampling_ratio/min": 0.747710257768631,
|
|
"sampling/sampling_logp_difference/max": 0.3071512818336487,
|
|
"sampling/sampling_logp_difference/mean": 0.021728925220668317,
|
|
"step": 5790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 355.1,
|
|
"completions/max_terminated_length": 355.1,
|
|
"completions/mean_length": 236.575,
|
|
"completions/mean_terminated_length": 236.575,
|
|
"completions/min_length": 154.9,
|
|
"completions/min_terminated_length": 154.9,
|
|
"entropy": 1.0457797840237617,
|
|
"epoch": 0.5332352670773192,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11066441237926483,
|
|
"kl": 0.04156140626873821,
|
|
"learning_rate": 4.7803671015995875e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 25147110.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.35502841174602506,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.3550284177064896,
|
|
"sampling/importance_sampling_ratio/max": 1.3520105481147766,
|
|
"sampling/importance_sampling_ratio/mean": 1.0007465183734894,
|
|
"sampling/importance_sampling_ratio/min": 0.7489111661911011,
|
|
"sampling/sampling_logp_difference/max": 0.32891182899475097,
|
|
"sampling/sampling_logp_difference/mean": 0.021299775503575803,
|
|
"step": 5800
|
|
},
|
|
{
|
|
"epoch": 0.5332352670773192,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002399650959860384,
|
|
"eval_completions/max_length": 311.3176265270506,
|
|
"eval_completions/max_terminated_length": 308.87085514834206,
|
|
"eval_completions/mean_length": 214.93106457242584,
|
|
"eval_completions/mean_terminated_length": 214.31188211224674,
|
|
"eval_completions/min_length": 138.0349040139616,
|
|
"eval_completions/min_terminated_length": 138.0349040139616,
|
|
"eval_entropy": 1.105723311451717,
|
|
"eval_frac_reward_zero_std": 0.2513089005235602,
|
|
"eval_kl": 0.0404926140713026,
|
|
"eval_loss": 0.002022106433287263,
|
|
"eval_num_tokens": 25147110.0,
|
|
"eval_reward": 0.0462478184991274,
|
|
"eval_reward_std": 0.3782694250293219,
|
|
"eval_rewards/reward_fn/mean": 0.0462478184991274,
|
|
"eval_rewards/reward_fn/std": 0.37826943709588173,
|
|
"eval_runtime": 1805.3312,
|
|
"eval_samples_per_second": 0.317,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2700951155567668,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999895534798321,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7736060693209916,
|
|
"eval_sampling/sampling_logp_difference/max": 0.283167751463712,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021514873132499723,
|
|
"eval_steps_per_second": 0.04,
|
|
"step": 5800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 311.1,
|
|
"completions/max_terminated_length": 301.1,
|
|
"completions/mean_length": 212.525,
|
|
"completions/mean_terminated_length": 210.09285888671874,
|
|
"completions/min_length": 130.5,
|
|
"completions/min_terminated_length": 130.5,
|
|
"entropy": 1.0477978996932507,
|
|
"epoch": 0.5341546382274525,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.900361061096191,
|
|
"kl": 0.05265176862012595,
|
|
"learning_rate": 4.779327490476068e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 25189528.0,
|
|
"reward": -0.09375,
|
|
"reward_std": 0.4678551137447357,
|
|
"rewards/reward_fn/mean": -0.09375,
|
|
"rewards/reward_fn/std": 0.4678551137447357,
|
|
"sampling/importance_sampling_ratio/max": 1.2737028479576111,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002426564693452,
|
|
"sampling/importance_sampling_ratio/min": 0.7759451687335968,
|
|
"sampling/sampling_logp_difference/max": 0.27017974853515625,
|
|
"sampling/sampling_logp_difference/mean": 0.021615360863506794,
|
|
"step": 5810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.5,
|
|
"completions/max_terminated_length": 324.5,
|
|
"completions/mean_length": 219.2125,
|
|
"completions/mean_terminated_length": 219.2125,
|
|
"completions/min_length": 137.2,
|
|
"completions/min_terminated_length": 137.2,
|
|
"entropy": 1.0800211794674397,
|
|
"epoch": 0.5350740093775858,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.12037783861160278,
|
|
"kl": 0.032971079961862416,
|
|
"learning_rate": 4.778285538300505e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 25235785.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.3899959743022919,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.38999598622322085,
|
|
"sampling/importance_sampling_ratio/max": 1.2625499725341798,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999870717525482,
|
|
"sampling/importance_sampling_ratio/min": 0.782296234369278,
|
|
"sampling/sampling_logp_difference/max": 0.2733001708984375,
|
|
"sampling/sampling_logp_difference/mean": 0.021053713746368885,
|
|
"step": 5820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 345.7,
|
|
"completions/max_terminated_length": 326.3,
|
|
"completions/mean_length": 226.575,
|
|
"completions/mean_terminated_length": 223.30892944335938,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 1.1620180413126946,
|
|
"epoch": 0.535993380527719,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.10908669233322144,
|
|
"kl": 0.0500280971522443,
|
|
"learning_rate": 4.777241246143066e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 25282247.0,
|
|
"reward": -0.11875,
|
|
"reward_std": 0.40702066421508787,
|
|
"rewards/reward_fn/mean": -0.11875,
|
|
"rewards/reward_fn/std": 0.4070206701755524,
|
|
"sampling/importance_sampling_ratio/max": 1.3046109199523925,
|
|
"sampling/importance_sampling_ratio/mean": 1.000963270664215,
|
|
"sampling/importance_sampling_ratio/min": 0.7893401265144349,
|
|
"sampling/sampling_logp_difference/max": 0.27059038877487185,
|
|
"sampling/sampling_logp_difference/mean": 0.022490573488175868,
|
|
"step": 5830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.9,
|
|
"completions/max_terminated_length": 271.9,
|
|
"completions/mean_length": 193.2875,
|
|
"completions/mean_terminated_length": 193.2875,
|
|
"completions/min_length": 124.2,
|
|
"completions/min_terminated_length": 124.2,
|
|
"entropy": 1.058194574713707,
|
|
"epoch": 0.5369127516778524,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.039623260498047,
|
|
"kl": 0.040199279098305854,
|
|
"learning_rate": 4.776194615076325e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 25322894.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.5242056578397751,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.5242056638002396,
|
|
"sampling/importance_sampling_ratio/max": 1.240907084941864,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999734878540039,
|
|
"sampling/importance_sampling_ratio/min": 0.7868142783641815,
|
|
"sampling/sampling_logp_difference/max": 0.2528844833374023,
|
|
"sampling/sampling_logp_difference/mean": 0.020994765870273113,
|
|
"step": 5840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.0,
|
|
"completions/max_terminated_length": 279.0,
|
|
"completions/mean_length": 196.575,
|
|
"completions/mean_terminated_length": 196.575,
|
|
"completions/min_length": 118.5,
|
|
"completions/min_terminated_length": 118.5,
|
|
"entropy": 1.0126894980669021,
|
|
"epoch": 0.5378321228279856,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.705118656158447,
|
|
"kl": 0.044608468236401674,
|
|
"learning_rate": 4.7751456461752545e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 25364932.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.3590047687292099,
|
|
"rewards/reward_fn/mean": 0.4625,
|
|
"rewards/reward_fn/std": 0.3590047687292099,
|
|
"sampling/importance_sampling_ratio/max": 1.2606943249702454,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998015284538269,
|
|
"sampling/importance_sampling_ratio/min": 0.767030942440033,
|
|
"sampling/sampling_logp_difference/max": 0.27470220923423766,
|
|
"sampling/sampling_logp_difference/mean": 0.021057704649865627,
|
|
"step": 5850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.5,
|
|
"completions/max_terminated_length": 281.5,
|
|
"completions/mean_length": 201.9375,
|
|
"completions/mean_terminated_length": 201.9375,
|
|
"completions/min_length": 135.9,
|
|
"completions/min_terminated_length": 135.9,
|
|
"entropy": 0.986550921201706,
|
|
"epoch": 0.538751493978119,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.23849630355835,
|
|
"kl": 0.04298890708014369,
|
|
"learning_rate": 4.7740943405172305e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 25409879.0,
|
|
"reward": 0.24375,
|
|
"reward_std": 0.4256551146507263,
|
|
"rewards/reward_fn/mean": 0.24375,
|
|
"rewards/reward_fn/std": 0.42565513253211973,
|
|
"sampling/importance_sampling_ratio/max": 1.2425489902496338,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001308917999268,
|
|
"sampling/importance_sampling_ratio/min": 0.7767520606517792,
|
|
"sampling/sampling_logp_difference/max": 0.26804039478302,
|
|
"sampling/sampling_logp_difference/mean": 0.020688995160162447,
|
|
"step": 5860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.0,
|
|
"completions/max_terminated_length": 293.0,
|
|
"completions/mean_length": 204.3375,
|
|
"completions/mean_terminated_length": 204.3375,
|
|
"completions/min_length": 129.6,
|
|
"completions/min_terminated_length": 129.6,
|
|
"entropy": 1.040500393509865,
|
|
"epoch": 0.5396708651282522,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.9332075119018555,
|
|
"kl": 0.032301264436682686,
|
|
"learning_rate": 4.773040699182028e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 25452218.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.4052957743406296,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.4052957743406296,
|
|
"sampling/importance_sampling_ratio/max": 1.2287989854812622,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999854445457459,
|
|
"sampling/importance_sampling_ratio/min": 0.7783948719501496,
|
|
"sampling/sampling_logp_difference/max": 0.258540940284729,
|
|
"sampling/sampling_logp_difference/mean": 0.02116065640002489,
|
|
"step": 5870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.7,
|
|
"completions/max_terminated_length": 279.7,
|
|
"completions/mean_length": 193.1875,
|
|
"completions/mean_terminated_length": 193.1875,
|
|
"completions/min_length": 132.4,
|
|
"completions/min_terminated_length": 132.4,
|
|
"entropy": 1.0589107371866704,
|
|
"epoch": 0.5405902362783855,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.18212489783763885,
|
|
"kl": 0.03585461411857978,
|
|
"learning_rate": 4.77198472325182e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 25491017.0,
|
|
"reward": 0.18125,
|
|
"reward_std": 0.42215809524059295,
|
|
"rewards/reward_fn/mean": 0.18125,
|
|
"rewards/reward_fn/std": 0.4221581071615219,
|
|
"sampling/importance_sampling_ratio/max": 1.2426268339157105,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000000834465026,
|
|
"sampling/importance_sampling_ratio/min": 0.776694244146347,
|
|
"sampling/sampling_logp_difference/max": 0.26067568063735963,
|
|
"sampling/sampling_logp_difference/mean": 0.02216607313603163,
|
|
"step": 5880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 256.6,
|
|
"completions/max_terminated_length": 256.6,
|
|
"completions/mean_length": 188.675,
|
|
"completions/mean_terminated_length": 188.675,
|
|
"completions/min_length": 125.2,
|
|
"completions/min_terminated_length": 125.2,
|
|
"entropy": 1.1061671853065491,
|
|
"epoch": 0.541509607428519,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.681638717651367,
|
|
"kl": 0.034587679780088365,
|
|
"learning_rate": 4.77092641381118e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 25531975.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3519521474838257,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.35195215344429015,
|
|
"sampling/importance_sampling_ratio/max": 1.2652634382247925,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998222529888153,
|
|
"sampling/importance_sampling_ratio/min": 0.7833308219909668,
|
|
"sampling/sampling_logp_difference/max": 0.29273965358734133,
|
|
"sampling/sampling_logp_difference/mean": 0.02201755326241255,
|
|
"step": 5890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.1,
|
|
"completions/max_terminated_length": 286.1,
|
|
"completions/mean_length": 193.3625,
|
|
"completions/mean_terminated_length": 193.3625,
|
|
"completions/min_length": 123.4,
|
|
"completions/min_terminated_length": 123.4,
|
|
"entropy": 1.048141146451235,
|
|
"epoch": 0.5424289785786522,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.654055595397949,
|
|
"kl": 0.05220045290188864,
|
|
"learning_rate": 4.769865771947076e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 25571260.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.5707158267498016,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.5707158446311951,
|
|
"sampling/importance_sampling_ratio/max": 1.2761346578598023,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999378561973572,
|
|
"sampling/importance_sampling_ratio/min": 0.7819115936756134,
|
|
"sampling/sampling_logp_difference/max": 0.26040356755256655,
|
|
"sampling/sampling_logp_difference/mean": 0.021015533432364463,
|
|
"step": 5900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.8,
|
|
"completions/max_terminated_length": 299.8,
|
|
"completions/mean_length": 209.275,
|
|
"completions/mean_terminated_length": 209.275,
|
|
"completions/min_length": 125.8,
|
|
"completions/min_terminated_length": 125.8,
|
|
"entropy": 1.1307473748922348,
|
|
"epoch": 0.5433483497287855,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.13110964000225067,
|
|
"kl": 0.030620167835149915,
|
|
"learning_rate": 4.768802798748871e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 25615010.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.44991534650325776,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4499153643846512,
|
|
"sampling/importance_sampling_ratio/max": 1.286659586429596,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997970759868622,
|
|
"sampling/importance_sampling_ratio/min": 0.7661869466304779,
|
|
"sampling/sampling_logp_difference/max": 0.280324125289917,
|
|
"sampling/sampling_logp_difference/mean": 0.02235572077333927,
|
|
"step": 5910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.9,
|
|
"completions/max_terminated_length": 297.9,
|
|
"completions/mean_length": 212.6125,
|
|
"completions/mean_terminated_length": 212.6125,
|
|
"completions/min_length": 133.3,
|
|
"completions/min_terminated_length": 133.3,
|
|
"entropy": 1.119990562647581,
|
|
"epoch": 0.5442677208789188,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11504147201776505,
|
|
"kl": 0.05544349694391713,
|
|
"learning_rate": 4.767737495308327e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 25658131.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.2903633385896683,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.2903633564710617,
|
|
"sampling/importance_sampling_ratio/max": 1.2705029726028443,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001543819904328,
|
|
"sampling/importance_sampling_ratio/min": 0.7840491950511932,
|
|
"sampling/sampling_logp_difference/max": 0.259272837638855,
|
|
"sampling/sampling_logp_difference/mean": 0.021829812228679656,
|
|
"step": 5920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 308.3,
|
|
"completions/max_terminated_length": 300.2,
|
|
"completions/mean_length": 218.7,
|
|
"completions/mean_terminated_length": 216.0464294433594,
|
|
"completions/min_length": 148.3,
|
|
"completions/min_terminated_length": 148.3,
|
|
"entropy": 1.1391092754900456,
|
|
"epoch": 0.5451870920290521,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.687870979309082,
|
|
"kl": 0.049077473138459025,
|
|
"learning_rate": 4.766669862719593e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 25705059.0,
|
|
"reward": -0.01875,
|
|
"reward_std": 0.5362760663032532,
|
|
"rewards/reward_fn/mean": -0.01875,
|
|
"rewards/reward_fn/std": 0.5362760841846466,
|
|
"sampling/importance_sampling_ratio/max": 1.3044365406036378,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996832191944123,
|
|
"sampling/importance_sampling_ratio/min": 0.7870841383934021,
|
|
"sampling/sampling_logp_difference/max": 0.2838757514953613,
|
|
"sampling/sampling_logp_difference/mean": 0.021945148706436157,
|
|
"step": 5930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 364.0,
|
|
"completions/max_terminated_length": 364.0,
|
|
"completions/mean_length": 236.55,
|
|
"completions/mean_terminated_length": 236.55,
|
|
"completions/min_length": 157.8,
|
|
"completions/min_terminated_length": 157.8,
|
|
"entropy": 1.0853421695530414,
|
|
"epoch": 0.5461064631791854,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.322802543640137,
|
|
"kl": 0.043733903742395344,
|
|
"learning_rate": 4.765599902079214e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 25752031.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5031163722276688,
|
|
"rewards/reward_fn/mean": 0.15625,
|
|
"rewards/reward_fn/std": 0.5031163841485977,
|
|
"sampling/importance_sampling_ratio/max": 1.2658527612686157,
|
|
"sampling/importance_sampling_ratio/mean": 1.000313937664032,
|
|
"sampling/importance_sampling_ratio/min": 0.7771980404853821,
|
|
"sampling/sampling_logp_difference/max": 0.2732072353363037,
|
|
"sampling/sampling_logp_difference/mean": 0.02070043422281742,
|
|
"step": 5940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.8,
|
|
"completions/max_terminated_length": 314.8,
|
|
"completions/mean_length": 215.4625,
|
|
"completions/mean_terminated_length": 215.4625,
|
|
"completions/min_length": 136.3,
|
|
"completions/min_terminated_length": 136.3,
|
|
"entropy": 1.0143737901002168,
|
|
"epoch": 0.5470258343293187,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.27232256531715393,
|
|
"kl": 0.044538831780664624,
|
|
"learning_rate": 4.764527614486125e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 25794580.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.4342138558626175,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.4342138677835464,
|
|
"sampling/importance_sampling_ratio/max": 1.308932363986969,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002804160118104,
|
|
"sampling/importance_sampling_ratio/min": 0.748006272315979,
|
|
"sampling/sampling_logp_difference/max": 0.33436107635498047,
|
|
"sampling/sampling_logp_difference/mean": 0.021697475761175155,
|
|
"step": 5950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 328.9,
|
|
"completions/max_terminated_length": 328.9,
|
|
"completions/mean_length": 236.9125,
|
|
"completions/mean_terminated_length": 236.9125,
|
|
"completions/min_length": 176.9,
|
|
"completions/min_terminated_length": 176.9,
|
|
"entropy": 1.0702579751610757,
|
|
"epoch": 0.547945205479452,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 4.59385871887207,
|
|
"kl": 0.04439568083034828,
|
|
"learning_rate": 4.763453001041653e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 25842317.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.2692759811878204,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.26927598714828493,
|
|
"sampling/importance_sampling_ratio/max": 1.2691372513771058,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001939117908478,
|
|
"sampling/importance_sampling_ratio/min": 0.7894082248210907,
|
|
"sampling/sampling_logp_difference/max": 0.26476741433143614,
|
|
"sampling/sampling_logp_difference/mean": 0.021000272035598753,
|
|
"step": 5960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.5,
|
|
"completions/max_terminated_length": 324.5,
|
|
"completions/mean_length": 221.65,
|
|
"completions/mean_terminated_length": 221.65,
|
|
"completions/min_length": 158.4,
|
|
"completions/min_terminated_length": 158.4,
|
|
"entropy": 1.1069272071123124,
|
|
"epoch": 0.5488645766295853,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.975583076477051,
|
|
"kl": 0.044007860706187785,
|
|
"learning_rate": 4.762376062849511e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 25888185.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.36538600325584414,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.36538602113723756,
|
|
"sampling/importance_sampling_ratio/max": 1.2860293626785277,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999909639358521,
|
|
"sampling/importance_sampling_ratio/min": 0.788635003566742,
|
|
"sampling/sampling_logp_difference/max": 0.27637232542037965,
|
|
"sampling/sampling_logp_difference/mean": 0.021719546616077424,
|
|
"step": 5970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.0,
|
|
"completions/max_terminated_length": 298.0,
|
|
"completions/mean_length": 211.5375,
|
|
"completions/mean_terminated_length": 211.5375,
|
|
"completions/min_length": 142.3,
|
|
"completions/min_terminated_length": 142.3,
|
|
"entropy": 1.1183372132480145,
|
|
"epoch": 0.5497839477797186,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.9230475425720215,
|
|
"kl": 0.047760227014077825,
|
|
"learning_rate": 4.7612968010158e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 25932444.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3452198475599289,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.3452198475599289,
|
|
"sampling/importance_sampling_ratio/max": 1.334183931350708,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001567363739015,
|
|
"sampling/importance_sampling_ratio/min": 0.7768688678741456,
|
|
"sampling/sampling_logp_difference/max": 0.2979376554489136,
|
|
"sampling/sampling_logp_difference/mean": 0.02282101195305586,
|
|
"step": 5980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.3,
|
|
"completions/max_terminated_length": 284.3,
|
|
"completions/mean_length": 213.45,
|
|
"completions/mean_terminated_length": 213.45,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 1.019435753673315,
|
|
"epoch": 0.550703318929852,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14074622094631195,
|
|
"kl": 0.04074702991638333,
|
|
"learning_rate": 4.760215216649009e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 25975888.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3818357288837433,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.38183574080467225,
|
|
"sampling/importance_sampling_ratio/max": 1.224917483329773,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000465393066407,
|
|
"sampling/importance_sampling_ratio/min": 0.7752545535564422,
|
|
"sampling/sampling_logp_difference/max": 0.2612292289733887,
|
|
"sampling/sampling_logp_difference/mean": 0.02056172639131546,
|
|
"step": 5990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 252.1,
|
|
"completions/max_terminated_length": 252.1,
|
|
"completions/mean_length": 185.575,
|
|
"completions/mean_terminated_length": 185.575,
|
|
"completions/min_length": 123.9,
|
|
"completions/min_terminated_length": 123.9,
|
|
"entropy": 0.9685670718550682,
|
|
"epoch": 0.5516226900799853,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.153370663523674,
|
|
"kl": 0.03268846122955438,
|
|
"learning_rate": 4.759131310860012e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 26018790.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.24315897822380067,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.2431589901447296,
|
|
"sampling/importance_sampling_ratio/max": 1.2644233584403992,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000719547271728,
|
|
"sampling/importance_sampling_ratio/min": 0.7764980137348175,
|
|
"sampling/sampling_logp_difference/max": 0.27544463872909547,
|
|
"sampling/sampling_logp_difference/mean": 0.0197725972160697,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"epoch": 0.5516226900799853,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 289.760907504363,
|
|
"eval_completions/max_terminated_length": 287.95113438045377,
|
|
"eval_completions/mean_length": 203.84293193717278,
|
|
"eval_completions/mean_terminated_length": 203.48245460283903,
|
|
"eval_completions/min_length": 133.26352530541013,
|
|
"eval_completions/min_terminated_length": 133.26352530541013,
|
|
"eval_entropy": 1.0827112896384161,
|
|
"eval_frac_reward_zero_std": 0.28272251308900526,
|
|
"eval_kl": 0.043951813903609596,
|
|
"eval_loss": 0.0022731476929038763,
|
|
"eval_num_tokens": 26018790.0,
|
|
"eval_reward": 0.04635689354275742,
|
|
"eval_reward_std": 0.36116907010943894,
|
|
"eval_rewards/reward_fn/mean": 0.04635689354275742,
|
|
"eval_rewards/reward_fn/std": 0.3611690798875133,
|
|
"eval_runtime": 1697.1728,
|
|
"eval_samples_per_second": 0.338,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2731469723982753,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999923838162713,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7796319361132477,
|
|
"eval_sampling/sampling_logp_difference/max": 0.278133765305108,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020720613938325572,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.3,
|
|
"completions/max_terminated_length": 307.3,
|
|
"completions/mean_length": 215.5375,
|
|
"completions/mean_terminated_length": 215.5375,
|
|
"completions/min_length": 137.5,
|
|
"completions/min_terminated_length": 137.5,
|
|
"entropy": 1.100583217293024,
|
|
"epoch": 0.5525420612301186,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.16121993958950043,
|
|
"kl": 0.04055323851061985,
|
|
"learning_rate": 4.758045084762068e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 26065633.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.1940147489309311,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.19401475489139558,
|
|
"sampling/importance_sampling_ratio/max": 1.2697038888931274,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998781323432923,
|
|
"sampling/importance_sampling_ratio/min": 0.7531592965126037,
|
|
"sampling/sampling_logp_difference/max": 0.32699408531188967,
|
|
"sampling/sampling_logp_difference/mean": 0.021225535683333872,
|
|
"step": 6010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 288.8,
|
|
"completions/max_terminated_length": 267.5,
|
|
"completions/mean_length": 195.2,
|
|
"completions/mean_terminated_length": 190.71428680419922,
|
|
"completions/min_length": 130.4,
|
|
"completions/min_terminated_length": 130.4,
|
|
"entropy": 0.9775846891105175,
|
|
"epoch": 0.5534614323802519,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.146791458129883,
|
|
"kl": 0.045046556543093176,
|
|
"learning_rate": 4.756956539470817e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 26112681.0,
|
|
"reward": 0.31875,
|
|
"reward_std": 0.2907280743122101,
|
|
"rewards/reward_fn/mean": 0.31875,
|
|
"rewards/reward_fn/std": 0.29072808027267455,
|
|
"sampling/importance_sampling_ratio/max": 1.2930991411209107,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000125825405122,
|
|
"sampling/importance_sampling_ratio/min": 0.7531468898057938,
|
|
"sampling/sampling_logp_difference/max": 0.34712321758270265,
|
|
"sampling/sampling_logp_difference/mean": 0.01998790940269828,
|
|
"step": 6020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 302.4,
|
|
"completions/max_terminated_length": 287.7,
|
|
"completions/mean_length": 215.7125,
|
|
"completions/mean_terminated_length": 212.2250015258789,
|
|
"completions/min_length": 143.5,
|
|
"completions/min_terminated_length": 143.5,
|
|
"entropy": 1.1139815695583821,
|
|
"epoch": 0.5543808035303852,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.11589720845222473,
|
|
"kl": 0.036113910312997176,
|
|
"learning_rate": 4.755865676104283e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 26153482.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3590047687292099,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.3590047687292099,
|
|
"sampling/importance_sampling_ratio/max": 1.2823933959007263,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002814829349518,
|
|
"sampling/importance_sampling_ratio/min": 0.7840506315231324,
|
|
"sampling/sampling_logp_difference/max": 0.28293814659118655,
|
|
"sampling/sampling_logp_difference/mean": 0.02164357490837574,
|
|
"step": 6030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.4,
|
|
"completions/max_terminated_length": 326.4,
|
|
"completions/mean_length": 206.5625,
|
|
"completions/mean_terminated_length": 206.5625,
|
|
"completions/min_length": 112.9,
|
|
"completions/min_terminated_length": 112.9,
|
|
"entropy": 1.0558997258543967,
|
|
"epoch": 0.5553001746805185,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.15067720413208,
|
|
"kl": 0.045980360300745815,
|
|
"learning_rate": 4.75477249578287e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 26197183.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.32997219264507294,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.32997219264507294,
|
|
"sampling/importance_sampling_ratio/max": 1.3412793040275575,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000888228416442,
|
|
"sampling/importance_sampling_ratio/min": 0.771163946390152,
|
|
"sampling/sampling_logp_difference/max": 0.305698561668396,
|
|
"sampling/sampling_logp_difference/mean": 0.020581517927348615,
|
|
"step": 6040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.2,
|
|
"completions/max_terminated_length": 316.2,
|
|
"completions/mean_length": 224.225,
|
|
"completions/mean_terminated_length": 224.225,
|
|
"completions/min_length": 141.7,
|
|
"completions/min_terminated_length": 141.7,
|
|
"entropy": 1.062789673358202,
|
|
"epoch": 0.5562195458306518,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.231579780578613,
|
|
"kl": 0.040888736740453166,
|
|
"learning_rate": 4.753676999629362e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 26239833.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.36674020886421205,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.3667402148246765,
|
|
"sampling/importance_sampling_ratio/max": 1.2890511751174927,
|
|
"sampling/importance_sampling_ratio/mean": 1.0007209539413453,
|
|
"sampling/importance_sampling_ratio/min": 0.774627935886383,
|
|
"sampling/sampling_logp_difference/max": 0.27654070854187013,
|
|
"sampling/sampling_logp_difference/mean": 0.021461549401283263,
|
|
"step": 6050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 318.1,
|
|
"completions/max_terminated_length": 318.1,
|
|
"completions/mean_length": 218.2375,
|
|
"completions/mean_terminated_length": 218.2375,
|
|
"completions/min_length": 145.4,
|
|
"completions/min_terminated_length": 145.4,
|
|
"entropy": 1.1033702749758958,
|
|
"epoch": 0.5571389169807851,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.480884075164795,
|
|
"kl": 0.05576675818301737,
|
|
"learning_rate": 4.752579188768921e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 26283268.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.3819254577159882,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.38192546367645264,
|
|
"sampling/importance_sampling_ratio/max": 1.2925401687622071,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996954083442688,
|
|
"sampling/importance_sampling_ratio/min": 0.782156103849411,
|
|
"sampling/sampling_logp_difference/max": 0.290268075466156,
|
|
"sampling/sampling_logp_difference/mean": 0.021014150511473418,
|
|
"step": 6060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.5,
|
|
"completions/max_terminated_length": 273.5,
|
|
"completions/mean_length": 199.4375,
|
|
"completions/mean_terminated_length": 199.4375,
|
|
"completions/min_length": 140.6,
|
|
"completions/min_terminated_length": 140.6,
|
|
"entropy": 1.0867134407162666,
|
|
"epoch": 0.5580582881309184,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.266357421875,
|
|
"kl": 0.05641554001485929,
|
|
"learning_rate": 4.7514790643290855e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 26325543.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.31855743527412417,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.3185574531555176,
|
|
"sampling/importance_sampling_ratio/max": 1.266220796108246,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006169736385346,
|
|
"sampling/importance_sampling_ratio/min": 0.7809510588645935,
|
|
"sampling/sampling_logp_difference/max": 0.2727139115333557,
|
|
"sampling/sampling_logp_difference/mean": 0.021806595474481584,
|
|
"step": 6070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.5,
|
|
"completions/max_terminated_length": 283.5,
|
|
"completions/mean_length": 207.3875,
|
|
"completions/mean_terminated_length": 207.3875,
|
|
"completions/min_length": 134.7,
|
|
"completions/min_terminated_length": 134.7,
|
|
"entropy": 1.0787999033927917,
|
|
"epoch": 0.5589776592810518,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.612980842590332,
|
|
"kl": 0.049633268325123936,
|
|
"learning_rate": 4.750376627439774e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 26367782.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.2903633385896683,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.2903633564710617,
|
|
"sampling/importance_sampling_ratio/max": 1.3411411046981812,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998104274272919,
|
|
"sampling/importance_sampling_ratio/min": 0.7889857649803161,
|
|
"sampling/sampling_logp_difference/max": 0.3039317846298218,
|
|
"sampling/sampling_logp_difference/mean": 0.021997787430882453,
|
|
"step": 6080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.1,
|
|
"completions/max_terminated_length": 298.1,
|
|
"completions/mean_length": 204.325,
|
|
"completions/mean_terminated_length": 204.325,
|
|
"completions/min_length": 134.2,
|
|
"completions/min_terminated_length": 134.2,
|
|
"entropy": 1.0672012146562337,
|
|
"epoch": 0.5598970304311851,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.694453716278076,
|
|
"kl": 0.0408846702426672,
|
|
"learning_rate": 4.749271879233276e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 26413360.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.3038551896810532,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.30385519564151764,
|
|
"sampling/importance_sampling_ratio/max": 1.2742053270339966,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997616529464721,
|
|
"sampling/importance_sampling_ratio/min": 0.7759513616561889,
|
|
"sampling/sampling_logp_difference/max": 0.2905306339263916,
|
|
"sampling/sampling_logp_difference/mean": 0.021338155213743448,
|
|
"step": 6090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.7,
|
|
"completions/max_terminated_length": 298.7,
|
|
"completions/mean_length": 215.9625,
|
|
"completions/mean_terminated_length": 215.9625,
|
|
"completions/min_length": 138.7,
|
|
"completions/min_terminated_length": 138.7,
|
|
"entropy": 1.067601116001606,
|
|
"epoch": 0.5608164015813184,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.920677185058594,
|
|
"kl": 0.04829503845539875,
|
|
"learning_rate": 4.748164820844257e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 26458293.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.5433805823326111,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.54338059425354,
|
|
"sampling/importance_sampling_ratio/max": 1.2721220016479493,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997223973274231,
|
|
"sampling/importance_sampling_ratio/min": 0.7419908940792084,
|
|
"sampling/sampling_logp_difference/max": 0.31930768489837646,
|
|
"sampling/sampling_logp_difference/mean": 0.021749875135719778,
|
|
"step": 6100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.2,
|
|
"completions/max_terminated_length": 310.2,
|
|
"completions/mean_length": 218.9125,
|
|
"completions/mean_terminated_length": 218.9125,
|
|
"completions/min_length": 136.7,
|
|
"completions/min_terminated_length": 136.7,
|
|
"entropy": 1.1123768582940101,
|
|
"epoch": 0.5617357727314517,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.11977187544107437,
|
|
"kl": 0.047653047821950165,
|
|
"learning_rate": 4.747055453409755e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 26506430.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.3492635846138,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.3492635846138,
|
|
"sampling/importance_sampling_ratio/max": 1.3172544240951538,
|
|
"sampling/importance_sampling_ratio/mean": 0.999568372964859,
|
|
"sampling/importance_sampling_ratio/min": 0.7611725032329559,
|
|
"sampling/sampling_logp_difference/max": 0.32140268087387086,
|
|
"sampling/sampling_logp_difference/mean": 0.022083052061498167,
|
|
"step": 6110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.7,
|
|
"completions/max_terminated_length": 319.7,
|
|
"completions/mean_length": 219.975,
|
|
"completions/mean_terminated_length": 219.975,
|
|
"completions/min_length": 154.1,
|
|
"completions/min_terminated_length": 154.1,
|
|
"entropy": 1.0652639500796794,
|
|
"epoch": 0.562655143881585,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.11650632321834564,
|
|
"kl": 0.04516810227651149,
|
|
"learning_rate": 4.745943778069177e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 26554532.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.5017905324697495,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.5017905324697495,
|
|
"sampling/importance_sampling_ratio/max": 1.2615557789802552,
|
|
"sampling/importance_sampling_ratio/mean": 1.000525599718094,
|
|
"sampling/importance_sampling_ratio/min": 0.7610154628753663,
|
|
"sampling/sampling_logp_difference/max": 0.29589412212371824,
|
|
"sampling/sampling_logp_difference/mean": 0.02028226088732481,
|
|
"step": 6120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 351.7,
|
|
"completions/max_terminated_length": 323.5,
|
|
"completions/mean_length": 223.1375,
|
|
"completions/mean_terminated_length": 219.0464294433594,
|
|
"completions/min_length": 124.9,
|
|
"completions/min_terminated_length": 124.9,
|
|
"entropy": 1.1002894632518292,
|
|
"epoch": 0.5635745150317183,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 4.478103160858154,
|
|
"kl": 0.04988232392934151,
|
|
"learning_rate": 4.744829795964306e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 26600975.0,
|
|
"reward": 0.24375,
|
|
"reward_std": 0.2298406779766083,
|
|
"rewards/reward_fn/mean": 0.24375,
|
|
"rewards/reward_fn/std": 0.2298406779766083,
|
|
"sampling/importance_sampling_ratio/max": 1.245114529132843,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993971467018128,
|
|
"sampling/importance_sampling_ratio/min": 0.7815256774425506,
|
|
"sampling/sampling_logp_difference/max": 0.2746048092842102,
|
|
"sampling/sampling_logp_difference/mean": 0.021487167477607726,
|
|
"step": 6130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.5,
|
|
"completions/max_terminated_length": 274.5,
|
|
"completions/mean_length": 190.6125,
|
|
"completions/mean_terminated_length": 190.6125,
|
|
"completions/min_length": 127.1,
|
|
"completions/min_terminated_length": 127.1,
|
|
"entropy": 1.1239974051713943,
|
|
"epoch": 0.5644938861818516,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1537046581506729,
|
|
"kl": 0.04689802597276867,
|
|
"learning_rate": 4.74371350823929e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 26643072.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.3138213872909546,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.31382139325141906,
|
|
"sampling/importance_sampling_ratio/max": 1.242549991607666,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996675312519073,
|
|
"sampling/importance_sampling_ratio/min": 0.7829691529273987,
|
|
"sampling/sampling_logp_difference/max": 0.25601160526275635,
|
|
"sampling/sampling_logp_difference/mean": 0.021785255521535873,
|
|
"step": 6140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 338.5,
|
|
"completions/max_terminated_length": 325.9,
|
|
"completions/mean_length": 231.1875,
|
|
"completions/mean_terminated_length": 227.95892944335938,
|
|
"completions/min_length": 154.4,
|
|
"completions/min_terminated_length": 154.4,
|
|
"entropy": 1.1363309763371945,
|
|
"epoch": 0.5654132573319849,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.242461681365967,
|
|
"kl": 0.04323768197791651,
|
|
"learning_rate": 4.742594916040644e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 26688415.0,
|
|
"reward": 0.24375,
|
|
"reward_std": 0.4626706510782242,
|
|
"rewards/reward_fn/mean": 0.24375,
|
|
"rewards/reward_fn/std": 0.46267067492008207,
|
|
"sampling/importance_sampling_ratio/max": 1.314042055606842,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001779854297639,
|
|
"sampling/importance_sampling_ratio/min": 0.7805741727352142,
|
|
"sampling/sampling_logp_difference/max": 0.2798584461212158,
|
|
"sampling/sampling_logp_difference/mean": 0.021532920375466346,
|
|
"step": 6150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.7,
|
|
"completions/max_terminated_length": 288.7,
|
|
"completions/mean_length": 209.875,
|
|
"completions/mean_terminated_length": 209.875,
|
|
"completions/min_length": 131.8,
|
|
"completions/min_terminated_length": 131.8,
|
|
"entropy": 1.0838081322610378,
|
|
"epoch": 0.5663326284821182,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.12752386927604675,
|
|
"kl": 0.03578949624206871,
|
|
"learning_rate": 4.741474020517254e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 26732805.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.2150476098060608,
|
|
"sampling/importance_sampling_ratio/max": 1.2556844830513,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003502786159515,
|
|
"sampling/importance_sampling_ratio/min": 0.7910115361213684,
|
|
"sampling/sampling_logp_difference/max": 0.2553782343864441,
|
|
"sampling/sampling_logp_difference/mean": 0.021159238554537295,
|
|
"step": 6160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 322.3,
|
|
"completions/max_terminated_length": 322.3,
|
|
"completions/mean_length": 209.4375,
|
|
"completions/mean_terminated_length": 209.4375,
|
|
"completions/min_length": 117.9,
|
|
"completions/min_terminated_length": 117.9,
|
|
"entropy": 1.0653237245976925,
|
|
"epoch": 0.5672519996322516,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.03389310836792,
|
|
"kl": 0.04591074122581631,
|
|
"learning_rate": 4.7403508228203684e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 26777632.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.29291952252388,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.2929195284843445,
|
|
"sampling/importance_sampling_ratio/max": 1.2589402079582215,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000591278076172,
|
|
"sampling/importance_sampling_ratio/min": 0.7985383152961731,
|
|
"sampling/sampling_logp_difference/max": 0.2497469425201416,
|
|
"sampling/sampling_logp_difference/mean": 0.021471216157078744,
|
|
"step": 6170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.0,
|
|
"completions/max_terminated_length": 281.0,
|
|
"completions/mean_length": 210.5625,
|
|
"completions/mean_terminated_length": 210.5625,
|
|
"completions/min_length": 145.0,
|
|
"completions/min_terminated_length": 145.0,
|
|
"entropy": 1.0731035716831685,
|
|
"epoch": 0.5681713707823849,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.894872665405273,
|
|
"kl": 0.05273802954470739,
|
|
"learning_rate": 4.739225324103601e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 26822989.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.28029437065124513,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.28029437065124513,
|
|
"sampling/importance_sampling_ratio/max": 1.2640633821487426,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004974901676178,
|
|
"sampling/importance_sampling_ratio/min": 0.7798752546310425,
|
|
"sampling/sampling_logp_difference/max": 0.2633428066968918,
|
|
"sampling/sampling_logp_difference/mean": 0.020562943629920484,
|
|
"step": 6180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 333.5,
|
|
"completions/max_terminated_length": 333.5,
|
|
"completions/mean_length": 240.5125,
|
|
"completions/mean_terminated_length": 240.5125,
|
|
"completions/min_length": 169.9,
|
|
"completions/min_terminated_length": 169.9,
|
|
"entropy": 1.1044846460223199,
|
|
"epoch": 0.5690907419325182,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.202932357788086,
|
|
"kl": 0.05687112370505929,
|
|
"learning_rate": 4.738097525522931e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 26870166.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.3409078598022461,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.3409078598022461,
|
|
"sampling/importance_sampling_ratio/max": 1.2937981963157654,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995344758033753,
|
|
"sampling/importance_sampling_ratio/min": 0.7897027254104614,
|
|
"sampling/sampling_logp_difference/max": 0.27732162475585936,
|
|
"sampling/sampling_logp_difference/mean": 0.0214335422962904,
|
|
"step": 6190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 290.2,
|
|
"completions/max_terminated_length": 278.9,
|
|
"completions/mean_length": 216.0,
|
|
"completions/mean_terminated_length": 213.3589294433594,
|
|
"completions/min_length": 145.2,
|
|
"completions/min_terminated_length": 145.2,
|
|
"entropy": 1.1585246749222278,
|
|
"epoch": 0.5700101130826515,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.032841682434082,
|
|
"kl": 0.05120847553480416,
|
|
"learning_rate": 4.736967428236697e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 26914574.0,
|
|
"reward": -0.09375,
|
|
"reward_std": 0.4978039592504501,
|
|
"rewards/reward_fn/mean": -0.09375,
|
|
"rewards/reward_fn/std": 0.4978039711713791,
|
|
"sampling/importance_sampling_ratio/max": 1.2411821842193604,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998735904693603,
|
|
"sampling/importance_sampling_ratio/min": 0.7408751308917999,
|
|
"sampling/sampling_logp_difference/max": 0.3060741126537323,
|
|
"sampling/sampling_logp_difference/mean": 0.022881741262972356,
|
|
"step": 6200
|
|
},
|
|
{
|
|
"epoch": 0.5700101130826515,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0004363001745200698,
|
|
"eval_completions/max_length": 294.5986038394415,
|
|
"eval_completions/max_terminated_length": 293.81849912739966,
|
|
"eval_completions/mean_length": 210.05388307155323,
|
|
"eval_completions/mean_terminated_length": 209.9302231820258,
|
|
"eval_completions/min_length": 139.92321116928446,
|
|
"eval_completions/min_terminated_length": 139.92321116928446,
|
|
"eval_entropy": 1.1115677457101265,
|
|
"eval_frac_reward_zero_std": 0.3106457242582897,
|
|
"eval_kl": 0.04608455646041956,
|
|
"eval_loss": 0.0023816365282982588,
|
|
"eval_num_tokens": 26914574.0,
|
|
"eval_reward": 0.0506108202443281,
|
|
"eval_reward_std": 0.33980271776725274,
|
|
"eval_rewards/reward_fn/mean": 0.0506108202443281,
|
|
"eval_rewards/reward_fn/std": 0.33980272723326094,
|
|
"eval_runtime": 1722.0475,
|
|
"eval_samples_per_second": 0.333,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2645804840440733,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999712471562531,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7805823519383842,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27004438565871686,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021134188845688656,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 6200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 312.9,
|
|
"completions/max_terminated_length": 292.7,
|
|
"completions/mean_length": 208.225,
|
|
"completions/mean_terminated_length": 204.40178680419922,
|
|
"completions/min_length": 132.8,
|
|
"completions/min_terminated_length": 132.8,
|
|
"entropy": 0.9808410197496414,
|
|
"epoch": 0.5709294842327848,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.011927604675293,
|
|
"kl": 0.04370331569225527,
|
|
"learning_rate": 4.735835033405598e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 26956656.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.36870584785938265,
|
|
"rewards/reward_fn/mean": 0.28125,
|
|
"rewards/reward_fn/std": 0.36870585978031156,
|
|
"sampling/importance_sampling_ratio/max": 1.2446932911872863,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005206286907196,
|
|
"sampling/importance_sampling_ratio/min": 0.8059887111186981,
|
|
"sampling/sampling_logp_difference/max": 0.23095146417617798,
|
|
"sampling/sampling_logp_difference/mean": 0.019828132819384336,
|
|
"step": 6210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.7,
|
|
"completions/max_terminated_length": 288.7,
|
|
"completions/mean_length": 205.725,
|
|
"completions/mean_terminated_length": 205.725,
|
|
"completions/min_length": 140.7,
|
|
"completions/min_terminated_length": 140.7,
|
|
"entropy": 1.1120463073253632,
|
|
"epoch": 0.5718488553829181,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.357625007629395,
|
|
"kl": 0.05083461130270735,
|
|
"learning_rate": 4.734700342192697e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 26999874.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.3110164314508438,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.3110164374113083,
|
|
"sampling/importance_sampling_ratio/max": 1.317823576927185,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000887751579284,
|
|
"sampling/importance_sampling_ratio/min": 0.7803912162780762,
|
|
"sampling/sampling_logp_difference/max": 0.2962776005268097,
|
|
"sampling/sampling_logp_difference/mean": 0.02169969454407692,
|
|
"step": 6220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.9,
|
|
"completions/max_terminated_length": 300.9,
|
|
"completions/mean_length": 212.7125,
|
|
"completions/mean_terminated_length": 212.7125,
|
|
"completions/min_length": 144.1,
|
|
"completions/min_terminated_length": 144.1,
|
|
"entropy": 1.1087385043501854,
|
|
"epoch": 0.5727682265330514,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1177276223897934,
|
|
"kl": 0.036686048936098815,
|
|
"learning_rate": 4.7335633557634105e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 27042963.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.4406357705593109,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.4406357824802399,
|
|
"sampling/importance_sampling_ratio/max": 1.2545434355735778,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999705791473389,
|
|
"sampling/importance_sampling_ratio/min": 0.7710280060768128,
|
|
"sampling/sampling_logp_difference/max": 0.283068859577179,
|
|
"sampling/sampling_logp_difference/mean": 0.02119218297302723,
|
|
"step": 6230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 356.4,
|
|
"completions/max_terminated_length": 343.8,
|
|
"completions/mean_length": 251.175,
|
|
"completions/mean_terminated_length": 248.21785888671874,
|
|
"completions/min_length": 138.1,
|
|
"completions/min_terminated_length": 138.1,
|
|
"entropy": 1.1300581611692906,
|
|
"epoch": 0.5736875976831847,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.12600478529930115,
|
|
"kl": 0.053841043740976605,
|
|
"learning_rate": 4.732424075285517e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 27092705.0,
|
|
"reward": 0.06875,
|
|
"reward_std": 0.27494093477725984,
|
|
"rewards/reward_fn/mean": 0.06875,
|
|
"rewards/reward_fn/std": 0.2749409407377243,
|
|
"sampling/importance_sampling_ratio/max": 1.2650659203529357,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998580515384674,
|
|
"sampling/importance_sampling_ratio/min": 0.7679423093795776,
|
|
"sampling/sampling_logp_difference/max": 0.2720667362213135,
|
|
"sampling/sampling_logp_difference/mean": 0.02121734358370304,
|
|
"step": 6240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.6,
|
|
"completions/max_terminated_length": 326.6,
|
|
"completions/mean_length": 227.65,
|
|
"completions/mean_terminated_length": 227.65,
|
|
"completions/min_length": 150.7,
|
|
"completions/min_terminated_length": 150.7,
|
|
"entropy": 1.1459137976169587,
|
|
"epoch": 0.574606968833318,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1364678293466568,
|
|
"kl": 0.0408228462212719,
|
|
"learning_rate": 4.731282501929145e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 27138957.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.34964423179626464,
|
|
"rewards/reward_fn/mean": 0.09375,
|
|
"rewards/reward_fn/std": 0.3496442437171936,
|
|
"sampling/importance_sampling_ratio/max": 1.2421517610549926,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999367952346802,
|
|
"sampling/importance_sampling_ratio/min": 0.7771855890750885,
|
|
"sampling/sampling_logp_difference/max": 0.2627410411834717,
|
|
"sampling/sampling_logp_difference/mean": 0.02135483454912901,
|
|
"step": 6250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.4,
|
|
"completions/max_terminated_length": 285.4,
|
|
"completions/mean_length": 215.1125,
|
|
"completions/mean_terminated_length": 215.1125,
|
|
"completions/min_length": 141.2,
|
|
"completions/min_terminated_length": 141.2,
|
|
"entropy": 1.0933591924607753,
|
|
"epoch": 0.5755263399834514,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.0477614402771,
|
|
"kl": 0.05091453074710443,
|
|
"learning_rate": 4.730138636866786e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 27184582.0,
|
|
"reward": 0.04375,
|
|
"reward_std": 0.3333195328712463,
|
|
"rewards/reward_fn/mean": 0.04375,
|
|
"rewards/reward_fn/std": 0.33331955075263975,
|
|
"sampling/importance_sampling_ratio/max": 1.2658711433410645,
|
|
"sampling/importance_sampling_ratio/mean": 1.000095695257187,
|
|
"sampling/importance_sampling_ratio/min": 0.7742916345596313,
|
|
"sampling/sampling_logp_difference/max": 0.27751768827438356,
|
|
"sampling/sampling_logp_difference/mean": 0.02102894466370344,
|
|
"step": 6260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.6,
|
|
"completions/max_terminated_length": 277.6,
|
|
"completions/mean_length": 192.5875,
|
|
"completions/mean_terminated_length": 192.5875,
|
|
"completions/min_length": 132.3,
|
|
"completions/min_terminated_length": 132.3,
|
|
"entropy": 1.0417233377695083,
|
|
"epoch": 0.5764457111335847,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.473278522491455,
|
|
"kl": 0.04627795366104692,
|
|
"learning_rate": 4.728992481273279e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 27225621.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.33088951706886294,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.33088952898979185,
|
|
"sampling/importance_sampling_ratio/max": 1.2925965547561646,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998872399330139,
|
|
"sampling/importance_sampling_ratio/min": 0.7827262163162232,
|
|
"sampling/sampling_logp_difference/max": 0.29452760219573976,
|
|
"sampling/sampling_logp_difference/mean": 0.02098462376743555,
|
|
"step": 6270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.0,
|
|
"completions/max_terminated_length": 294.0,
|
|
"completions/mean_length": 202.0,
|
|
"completions/mean_terminated_length": 202.0,
|
|
"completions/min_length": 139.2,
|
|
"completions/min_terminated_length": 139.2,
|
|
"entropy": 1.0964640878140925,
|
|
"epoch": 0.577365082283718,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1317955106496811,
|
|
"kl": 0.042411452380474654,
|
|
"learning_rate": 4.727844036325819e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 27269565.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.2912300378084183,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.2912300378084183,
|
|
"sampling/importance_sampling_ratio/max": 1.278913974761963,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997564315795898,
|
|
"sampling/importance_sampling_ratio/min": 0.7810963153839111,
|
|
"sampling/sampling_logp_difference/max": 0.282138466835022,
|
|
"sampling/sampling_logp_difference/mean": 0.021411254070699215,
|
|
"step": 6280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.8,
|
|
"completions/max_terminated_length": 268.8,
|
|
"completions/mean_length": 194.3625,
|
|
"completions/mean_terminated_length": 194.3625,
|
|
"completions/min_length": 117.8,
|
|
"completions/min_terminated_length": 117.8,
|
|
"entropy": 1.047000838816166,
|
|
"epoch": 0.5782844534338513,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.521574020385742,
|
|
"kl": 0.047210085089318454,
|
|
"learning_rate": 4.726693303203949e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 27313458.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.36765356361866,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.36765357553958894,
|
|
"sampling/importance_sampling_ratio/max": 1.2391393065452576,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995747983455658,
|
|
"sampling/importance_sampling_ratio/min": 0.7767157316207886,
|
|
"sampling/sampling_logp_difference/max": 0.2723180651664734,
|
|
"sampling/sampling_logp_difference/mean": 0.020543034188449383,
|
|
"step": 6290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 263.6,
|
|
"completions/max_terminated_length": 263.6,
|
|
"completions/mean_length": 193.4,
|
|
"completions/mean_terminated_length": 193.4,
|
|
"completions/min_length": 130.5,
|
|
"completions/min_terminated_length": 130.5,
|
|
"entropy": 1.1493043258786202,
|
|
"epoch": 0.5792038245839846,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.613497257232666,
|
|
"kl": 0.049664326454512776,
|
|
"learning_rate": 4.725540283089566e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 27356082.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.43406798243522643,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.4340679943561554,
|
|
"sampling/importance_sampling_ratio/max": 1.2571020603179932,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005217492580414,
|
|
"sampling/importance_sampling_ratio/min": 0.8086998701095581,
|
|
"sampling/sampling_logp_difference/max": 0.2326374053955078,
|
|
"sampling/sampling_logp_difference/mean": 0.02168071996420622,
|
|
"step": 6300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.9,
|
|
"completions/max_terminated_length": 278.9,
|
|
"completions/mean_length": 191.075,
|
|
"completions/mean_terminated_length": 191.075,
|
|
"completions/min_length": 121.7,
|
|
"completions/min_terminated_length": 121.7,
|
|
"entropy": 1.1032161392271518,
|
|
"epoch": 0.5801231957341179,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.90437126159668,
|
|
"kl": 0.05803560609929263,
|
|
"learning_rate": 4.724384977166913e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 27397072.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.3558872640132904,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.3558872640132904,
|
|
"sampling/importance_sampling_ratio/max": 1.2864754796028137,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003357708454133,
|
|
"sampling/importance_sampling_ratio/min": 0.794737946987152,
|
|
"sampling/sampling_logp_difference/max": 0.25749757289886477,
|
|
"sampling/sampling_logp_difference/mean": 0.021868198551237582,
|
|
"step": 6310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 301.3,
|
|
"completions/max_terminated_length": 290.5,
|
|
"completions/mean_length": 220.425,
|
|
"completions/mean_terminated_length": 214.7916687011719,
|
|
"completions/min_length": 130.4,
|
|
"completions/min_terminated_length": 130.4,
|
|
"entropy": 1.0283865824341774,
|
|
"epoch": 0.5810425668842512,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.22281497716903687,
|
|
"kl": 0.053705842152703555,
|
|
"learning_rate": 4.723227386622583e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 27439946.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.3954332768917084,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.3954332947731018,
|
|
"sampling/importance_sampling_ratio/max": 1.2724909424781798,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998671650886536,
|
|
"sampling/importance_sampling_ratio/min": 0.7961435317993164,
|
|
"sampling/sampling_logp_difference/max": 0.25638039112091066,
|
|
"sampling/sampling_logp_difference/mean": 0.020488291792571545,
|
|
"step": 6320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 323.0,
|
|
"completions/max_terminated_length": 319.7,
|
|
"completions/mean_length": 214.7875,
|
|
"completions/mean_terminated_length": 212.5625,
|
|
"completions/min_length": 131.6,
|
|
"completions/min_terminated_length": 131.6,
|
|
"entropy": 1.0846950247883798,
|
|
"epoch": 0.5819619380343845,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1487865298986435,
|
|
"kl": 0.062440075573977086,
|
|
"learning_rate": 4.722067512645512e-06,
|
|
"loss": 0.0035,
|
|
"num_tokens": 27488017.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.23616344034671782,
|
|
"rewards/reward_fn/mean": 0.03125,
|
|
"rewards/reward_fn/std": 0.23616344034671782,
|
|
"sampling/importance_sampling_ratio/max": 1.293822467327118,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998626828193664,
|
|
"sampling/importance_sampling_ratio/min": 0.7827783286571502,
|
|
"sampling/sampling_logp_difference/max": 0.29886641502380373,
|
|
"sampling/sampling_logp_difference/mean": 0.021185122057795526,
|
|
"step": 6330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 235.4,
|
|
"completions/max_terminated_length": 235.4,
|
|
"completions/mean_length": 167.425,
|
|
"completions/mean_terminated_length": 167.425,
|
|
"completions/min_length": 115.1,
|
|
"completions/min_terminated_length": 115.1,
|
|
"entropy": 0.9618630453944206,
|
|
"epoch": 0.5828813091845177,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.749384880065918,
|
|
"kl": 0.05407567550428212,
|
|
"learning_rate": 4.7209053564269854e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 27525035.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.34459956288337706,
|
|
"rewards/reward_fn/mean": 0.2875,
|
|
"rewards/reward_fn/std": 0.3445995688438416,
|
|
"sampling/importance_sampling_ratio/max": 1.2630678057670592,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997841656208039,
|
|
"sampling/importance_sampling_ratio/min": 0.7938756883144379,
|
|
"sampling/sampling_logp_difference/max": 0.24610230922698975,
|
|
"sampling/sampling_logp_difference/mean": 0.019998835027217866,
|
|
"step": 6340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.4,
|
|
"completions/max_terminated_length": 273.4,
|
|
"completions/mean_length": 203.4875,
|
|
"completions/mean_terminated_length": 203.4875,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 1.0706591986119747,
|
|
"epoch": 0.583800680334651,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.13787001371383667,
|
|
"kl": 0.060292388102971016,
|
|
"learning_rate": 4.719740919160631e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 27567858.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.11700168251991272,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.11700168251991272,
|
|
"sampling/importance_sampling_ratio/max": 1.2388605713844298,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002129137516023,
|
|
"sampling/importance_sampling_ratio/min": 0.7904730856418609,
|
|
"sampling/sampling_logp_difference/max": 0.24790539741516113,
|
|
"sampling/sampling_logp_difference/mean": 0.020003251917660236,
|
|
"step": 6350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.9,
|
|
"completions/max_terminated_length": 291.9,
|
|
"completions/mean_length": 208.1,
|
|
"completions/mean_terminated_length": 208.1,
|
|
"completions/min_length": 136.1,
|
|
"completions/min_terminated_length": 136.1,
|
|
"entropy": 1.0725885674357414,
|
|
"epoch": 0.5847200514847845,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.065314292907715,
|
|
"kl": 0.05769384795567021,
|
|
"learning_rate": 4.718574202042419e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 27610594.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.332049286365509,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.3320492923259735,
|
|
"sampling/importance_sampling_ratio/max": 1.2333193302154541,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994445323944092,
|
|
"sampling/importance_sampling_ratio/min": 0.7887594103813171,
|
|
"sampling/sampling_logp_difference/max": 0.2447727680206299,
|
|
"sampling/sampling_logp_difference/mean": 0.02111423071473837,
|
|
"step": 6360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.5,
|
|
"completions/max_terminated_length": 325.5,
|
|
"completions/mean_length": 217.55,
|
|
"completions/mean_terminated_length": 217.55,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 1.0757221065461635,
|
|
"epoch": 0.5856394226349178,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.439064025878906,
|
|
"kl": 0.042228489054832605,
|
|
"learning_rate": 4.71740520627066e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 27654126.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3035428076982498,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.3035428076982498,
|
|
"sampling/importance_sampling_ratio/max": 1.2590969324111938,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998880386352539,
|
|
"sampling/importance_sampling_ratio/min": 0.7882028698921204,
|
|
"sampling/sampling_logp_difference/max": 0.25784212350845337,
|
|
"sampling/sampling_logp_difference/mean": 0.02142437845468521,
|
|
"step": 6370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 323.6,
|
|
"completions/max_terminated_length": 304.6,
|
|
"completions/mean_length": 222.675,
|
|
"completions/mean_terminated_length": 217.1857162475586,
|
|
"completions/min_length": 155.9,
|
|
"completions/min_terminated_length": 155.9,
|
|
"entropy": 1.1420228436589241,
|
|
"epoch": 0.586558793785051,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.10710560530424118,
|
|
"kl": 0.061873108858708295,
|
|
"learning_rate": 4.716233933046008e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 27698020.0,
|
|
"reward": -0.09375,
|
|
"reward_std": 0.29333876073360443,
|
|
"rewards/reward_fn/mean": -0.09375,
|
|
"rewards/reward_fn/std": 0.2933387666940689,
|
|
"sampling/importance_sampling_ratio/max": 1.2376533508300782,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000510334968566,
|
|
"sampling/importance_sampling_ratio/min": 0.7760604858398438,
|
|
"sampling/sampling_logp_difference/max": 0.2708033323287964,
|
|
"sampling/sampling_logp_difference/mean": 0.021824695356190204,
|
|
"step": 6380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.5,
|
|
"completions/max_terminated_length": 310.5,
|
|
"completions/mean_length": 226.7,
|
|
"completions/mean_terminated_length": 226.7,
|
|
"completions/min_length": 162.8,
|
|
"completions/min_terminated_length": 162.8,
|
|
"entropy": 1.1181060269474983,
|
|
"epoch": 0.5874781649351843,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1111498549580574,
|
|
"kl": 0.061604120791889726,
|
|
"learning_rate": 4.715060383571454e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 27745724.0,
|
|
"reward": -0.19375,
|
|
"reward_std": 0.3649082899093628,
|
|
"rewards/reward_fn/mean": -0.19375,
|
|
"rewards/reward_fn/std": 0.3649082899093628,
|
|
"sampling/importance_sampling_ratio/max": 1.2603531360626221,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997588336467743,
|
|
"sampling/importance_sampling_ratio/min": 0.791512793302536,
|
|
"sampling/sampling_logp_difference/max": 0.25716852247714994,
|
|
"sampling/sampling_logp_difference/mean": 0.021364770643413068,
|
|
"step": 6390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.7,
|
|
"completions/max_terminated_length": 288.7,
|
|
"completions/mean_length": 205.95,
|
|
"completions/mean_terminated_length": 205.95,
|
|
"completions/min_length": 127.6,
|
|
"completions/min_terminated_length": 127.6,
|
|
"entropy": 1.1478823773562907,
|
|
"epoch": 0.5883975360853176,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.890495777130127,
|
|
"kl": 0.06008486316422932,
|
|
"learning_rate": 4.713884559052327e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 27790392.0,
|
|
"reward": -0.225,
|
|
"reward_std": 0.37287638187408445,
|
|
"rewards/reward_fn/mean": -0.225,
|
|
"rewards/reward_fn/std": 0.37287638187408445,
|
|
"sampling/importance_sampling_ratio/max": 1.239621114730835,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000463724136353,
|
|
"sampling/importance_sampling_ratio/min": 0.7828689455986023,
|
|
"sampling/sampling_logp_difference/max": 0.2543454885482788,
|
|
"sampling/sampling_logp_difference/mean": 0.02227793224155903,
|
|
"step": 6400
|
|
},
|
|
{
|
|
"epoch": 0.5883975360853176,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.001963350785340314,
|
|
"eval_completions/max_length": 301.18848167539267,
|
|
"eval_completions/max_terminated_length": 299.4188481675393,
|
|
"eval_completions/mean_length": 212.71684118673647,
|
|
"eval_completions/mean_terminated_length": 212.20256814740299,
|
|
"eval_completions/min_length": 140.72076788830717,
|
|
"eval_completions/min_terminated_length": 140.72076788830717,
|
|
"eval_entropy": 1.1127406142352971,
|
|
"eval_frac_reward_zero_std": 0.42408376963350786,
|
|
"eval_kl": 0.06194145265921144,
|
|
"eval_loss": 0.003257643198594451,
|
|
"eval_num_tokens": 27790392.0,
|
|
"eval_reward": 0.05977312390924956,
|
|
"eval_reward_std": 0.26429427164179287,
|
|
"eval_rewards/reward_fn/mean": 0.05977312390924956,
|
|
"eval_rewards/reward_fn/std": 0.2642942788193155,
|
|
"eval_runtime": 1757.0686,
|
|
"eval_samples_per_second": 0.326,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2742895018992,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999519474219277,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.767305584894217,
|
|
"eval_sampling/sampling_logp_difference/max": 0.29101582737493265,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021767915346794623,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 6400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.6,
|
|
"completions/max_terminated_length": 281.6,
|
|
"completions/mean_length": 199.4,
|
|
"completions/mean_terminated_length": 199.4,
|
|
"completions/min_length": 129.5,
|
|
"completions/min_terminated_length": 129.5,
|
|
"entropy": 1.140552642941475,
|
|
"epoch": 0.5893169072354509,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.17851880192756653,
|
|
"kl": 0.06977051523281261,
|
|
"learning_rate": 4.712706460696292e-06,
|
|
"loss": 0.0037,
|
|
"num_tokens": 27831712.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.0980459213256836,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.09804592728614807,
|
|
"sampling/importance_sampling_ratio/max": 1.2868279099464417,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006411850452424,
|
|
"sampling/importance_sampling_ratio/min": 0.7834126949310303,
|
|
"sampling/sampling_logp_difference/max": 0.25857616364955904,
|
|
"sampling/sampling_logp_difference/mean": 0.021910130232572555,
|
|
"step": 6410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 391.3,
|
|
"completions/max_terminated_length": 374.3,
|
|
"completions/mean_length": 260.6,
|
|
"completions/mean_terminated_length": 253.42083435058595,
|
|
"completions/min_length": 171.9,
|
|
"completions/min_terminated_length": 171.9,
|
|
"entropy": 1.0753916244953872,
|
|
"epoch": 0.5902362783855842,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.11423100531101227,
|
|
"kl": 0.06381710111163556,
|
|
"learning_rate": 4.711526089713353e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 27880344.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.18643819093704223,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.18643819093704223,
|
|
"sampling/importance_sampling_ratio/max": 1.2943445324897767,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000058591365815,
|
|
"sampling/importance_sampling_ratio/min": 0.7858851313591003,
|
|
"sampling/sampling_logp_difference/max": 0.2742376565933228,
|
|
"sampling/sampling_logp_difference/mean": 0.020468858629465105,
|
|
"step": 6420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 311.0,
|
|
"completions/max_terminated_length": 295.2,
|
|
"completions/mean_length": 218.05,
|
|
"completions/mean_terminated_length": 215.0375,
|
|
"completions/min_length": 142.3,
|
|
"completions/min_terminated_length": 142.3,
|
|
"entropy": 1.0088297475129366,
|
|
"epoch": 0.5911556495357175,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.517147541046143,
|
|
"kl": 0.05601551482686773,
|
|
"learning_rate": 4.710343447315845e-06,
|
|
"loss": 0.0034,
|
|
"num_tokens": 27923540.0,
|
|
"reward": 0.05625,
|
|
"reward_std": 0.34339634478092196,
|
|
"rewards/reward_fn/mean": 0.05625,
|
|
"rewards/reward_fn/std": 0.3433963626623154,
|
|
"sampling/importance_sampling_ratio/max": 1.2495843410491942,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999912917613983,
|
|
"sampling/importance_sampling_ratio/min": 0.7804013431072235,
|
|
"sampling/sampling_logp_difference/max": 0.2558472752571106,
|
|
"sampling/sampling_logp_difference/mean": 0.020548405684530736,
|
|
"step": 6430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.8,
|
|
"completions/max_terminated_length": 283.8,
|
|
"completions/mean_length": 199.025,
|
|
"completions/mean_terminated_length": 199.025,
|
|
"completions/min_length": 134.1,
|
|
"completions/min_terminated_length": 134.1,
|
|
"entropy": 1.0597386829555036,
|
|
"epoch": 0.5920750206858508,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.834327220916748,
|
|
"kl": 0.04844910238170996,
|
|
"learning_rate": 4.709158534718435e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 27962574.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.3984315723180771,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.3984315901994705,
|
|
"sampling/importance_sampling_ratio/max": 1.2805827975273132,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006895005702972,
|
|
"sampling/importance_sampling_ratio/min": 0.790285712480545,
|
|
"sampling/sampling_logp_difference/max": 0.2671658992767334,
|
|
"sampling/sampling_logp_difference/mean": 0.02112551834434271,
|
|
"step": 6440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.3,
|
|
"completions/max_terminated_length": 273.3,
|
|
"completions/mean_length": 193.925,
|
|
"completions/mean_terminated_length": 193.925,
|
|
"completions/min_length": 126.9,
|
|
"completions/min_terminated_length": 126.9,
|
|
"entropy": 1.1002034984529019,
|
|
"epoch": 0.5929943918359842,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.945777893066406,
|
|
"kl": 0.06655430651153438,
|
|
"learning_rate": 4.707971353138124e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 28005328.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.42685834467411043,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.42685835659503935,
|
|
"sampling/importance_sampling_ratio/max": 1.2863455653190612,
|
|
"sampling/importance_sampling_ratio/mean": 1.000013881921768,
|
|
"sampling/importance_sampling_ratio/min": 0.7873491227626801,
|
|
"sampling/sampling_logp_difference/max": 0.28044370412826536,
|
|
"sampling/sampling_logp_difference/mean": 0.02188140116631985,
|
|
"step": 6450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.5,
|
|
"completions/max_terminated_length": 292.5,
|
|
"completions/mean_length": 210.1,
|
|
"completions/mean_terminated_length": 210.1,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 1.0388106063008309,
|
|
"epoch": 0.5939137629861175,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.690207004547119,
|
|
"kl": 0.049655258230632174,
|
|
"learning_rate": 4.706781903794243e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 28048352.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.34984527826309203,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.34984528422355654,
|
|
"sampling/importance_sampling_ratio/max": 1.2694766402244568,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999480605125427,
|
|
"sampling/importance_sampling_ratio/min": 0.7341262042522431,
|
|
"sampling/sampling_logp_difference/max": 0.34055122137069704,
|
|
"sampling/sampling_logp_difference/mean": 0.021230046078562735,
|
|
"step": 6460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 348.6,
|
|
"completions/max_terminated_length": 348.6,
|
|
"completions/mean_length": 239.1625,
|
|
"completions/mean_terminated_length": 239.1625,
|
|
"completions/min_length": 155.2,
|
|
"completions/min_terminated_length": 155.2,
|
|
"entropy": 1.124115265160799,
|
|
"epoch": 0.5948331341362508,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.366975784301758,
|
|
"kl": 0.04762445252854377,
|
|
"learning_rate": 4.705590187908452e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 28096581.0,
|
|
"reward": -0.325,
|
|
"reward_std": 0.2794355183839798,
|
|
"rewards/reward_fn/mean": -0.325,
|
|
"rewards/reward_fn/std": 0.2794355243444443,
|
|
"sampling/importance_sampling_ratio/max": 1.260988676548004,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001127541065216,
|
|
"sampling/importance_sampling_ratio/min": 0.7639222085475922,
|
|
"sampling/sampling_logp_difference/max": 0.28994319438934324,
|
|
"sampling/sampling_logp_difference/mean": 0.02184861060231924,
|
|
"step": 6470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.3,
|
|
"completions/max_terminated_length": 311.3,
|
|
"completions/mean_length": 213.95,
|
|
"completions/mean_terminated_length": 213.95,
|
|
"completions/min_length": 137.4,
|
|
"completions/min_terminated_length": 137.4,
|
|
"entropy": 1.1191321179270743,
|
|
"epoch": 0.5957525052863841,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.16259022057056427,
|
|
"kl": 0.06310149931232445,
|
|
"learning_rate": 4.7043962067047376e-06,
|
|
"loss": 0.0036,
|
|
"num_tokens": 28140729.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.2972315102815628,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.29723151624202726,
|
|
"sampling/importance_sampling_ratio/max": 1.2911235809326171,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000535488128661,
|
|
"sampling/importance_sampling_ratio/min": 0.7360935509204865,
|
|
"sampling/sampling_logp_difference/max": 0.343064284324646,
|
|
"sampling/sampling_logp_difference/mean": 0.02265188079327345,
|
|
"step": 6480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.5,
|
|
"completions/max_terminated_length": 295.5,
|
|
"completions/mean_length": 218.05,
|
|
"completions/mean_terminated_length": 218.05,
|
|
"completions/min_length": 143.7,
|
|
"completions/min_terminated_length": 143.7,
|
|
"entropy": 1.073278898000717,
|
|
"epoch": 0.5966718764365174,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.741918563842773,
|
|
"kl": 0.05609490211354569,
|
|
"learning_rate": 4.703199961409415e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 28186085.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3717166125774384,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.37171661853790283,
|
|
"sampling/importance_sampling_ratio/max": 1.3422470450401307,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004152178764343,
|
|
"sampling/importance_sampling_ratio/min": 0.7311745285987854,
|
|
"sampling/sampling_logp_difference/max": 0.3469825744628906,
|
|
"sampling/sampling_logp_difference/mean": 0.021227271854877473,
|
|
"step": 6490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.9,
|
|
"completions/max_terminated_length": 272.9,
|
|
"completions/mean_length": 210.7125,
|
|
"completions/mean_terminated_length": 210.7125,
|
|
"completions/min_length": 154.5,
|
|
"completions/min_terminated_length": 154.5,
|
|
"entropy": 1.1236042480915784,
|
|
"epoch": 0.5975912475866507,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.10843749344348907,
|
|
"kl": 0.06298108347691596,
|
|
"learning_rate": 4.702001453251125e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 28231518.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.2857582807540894,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.28575828671455383,
|
|
"sampling/importance_sampling_ratio/max": 1.259701633453369,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999745070934296,
|
|
"sampling/importance_sampling_ratio/min": 0.7881865620613098,
|
|
"sampling/sampling_logp_difference/max": 0.26497507095336914,
|
|
"sampling/sampling_logp_difference/mean": 0.021484932489693166,
|
|
"step": 6500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.0,
|
|
"completions/max_terminated_length": 258.0,
|
|
"completions/mean_length": 206.8875,
|
|
"completions/mean_terminated_length": 206.8875,
|
|
"completions/min_length": 147.4,
|
|
"completions/min_terminated_length": 147.4,
|
|
"entropy": 1.1007804326713084,
|
|
"epoch": 0.598510618736784,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.295722007751465,
|
|
"kl": 0.05527350713382475,
|
|
"learning_rate": 4.700800683460829e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 28274269.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.2748226135969162,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.2748226195573807,
|
|
"sampling/importance_sampling_ratio/max": 1.2431827545166017,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996847808361053,
|
|
"sampling/importance_sampling_ratio/min": 0.7552955746650696,
|
|
"sampling/sampling_logp_difference/max": 0.29806989431381226,
|
|
"sampling/sampling_logp_difference/mean": 0.02180723175406456,
|
|
"step": 6510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 346.4,
|
|
"completions/max_terminated_length": 330.1,
|
|
"completions/mean_length": 237.4,
|
|
"completions/mean_terminated_length": 232.0,
|
|
"completions/min_length": 155.8,
|
|
"completions/min_terminated_length": 155.8,
|
|
"entropy": 1.1116694692522286,
|
|
"epoch": 0.5994299898869173,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1328957974910736,
|
|
"kl": 0.0401865983963944,
|
|
"learning_rate": 4.699597653271816e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 28322285.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.3564689576625824,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.35646896362304686,
|
|
"sampling/importance_sampling_ratio/max": 1.32187876701355,
|
|
"sampling/importance_sampling_ratio/mean": 1.00042262673378,
|
|
"sampling/importance_sampling_ratio/min": 0.7949499130249024,
|
|
"sampling/sampling_logp_difference/max": 0.2899640560150146,
|
|
"sampling/sampling_logp_difference/mean": 0.021367712318897246,
|
|
"step": 6520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 304.4,
|
|
"completions/max_terminated_length": 296.8,
|
|
"completions/mean_length": 214.7875,
|
|
"completions/mean_terminated_length": 211.18035888671875,
|
|
"completions/min_length": 149.4,
|
|
"completions/min_terminated_length": 149.4,
|
|
"entropy": 1.0357746385037898,
|
|
"epoch": 0.6003493610370506,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.095525741577148,
|
|
"kl": 0.0541783868917264,
|
|
"learning_rate": 4.698392363919694e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 28366844.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.22814334630966188,
|
|
"rewards/reward_fn/mean": 0.03125,
|
|
"rewards/reward_fn/std": 0.22814335227012633,
|
|
"sampling/importance_sampling_ratio/max": 1.2725078105926513,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002647578716277,
|
|
"sampling/importance_sampling_ratio/min": 0.8044821918010712,
|
|
"sampling/sampling_logp_difference/max": 0.25182098150253296,
|
|
"sampling/sampling_logp_difference/mean": 0.02063723560422659,
|
|
"step": 6530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.5,
|
|
"completions/max_terminated_length": 299.5,
|
|
"completions/mean_length": 212.7,
|
|
"completions/mean_terminated_length": 212.7,
|
|
"completions/min_length": 143.8,
|
|
"completions/min_terminated_length": 143.8,
|
|
"entropy": 1.0229517601430416,
|
|
"epoch": 0.601268732187184,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.14948205649852753,
|
|
"kl": 0.04433594692964107,
|
|
"learning_rate": 4.697184816642392e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 28409660.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.45328023433685305,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.45328024625778196,
|
|
"sampling/importance_sampling_ratio/max": 1.2769538044929505,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999709725379944,
|
|
"sampling/importance_sampling_ratio/min": 0.7692527294158935,
|
|
"sampling/sampling_logp_difference/max": 0.2807143568992615,
|
|
"sampling/sampling_logp_difference/mean": 0.02112936656922102,
|
|
"step": 6540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 300.2,
|
|
"completions/max_terminated_length": 281.7,
|
|
"completions/mean_length": 214.8875,
|
|
"completions/mean_terminated_length": 211.7107147216797,
|
|
"completions/min_length": 146.9,
|
|
"completions/min_terminated_length": 146.9,
|
|
"entropy": 1.0688552245497704,
|
|
"epoch": 0.6021881033373173,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.147977352142334,
|
|
"kl": 0.06027426638174802,
|
|
"learning_rate": 4.6959750126801565e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 28455963.0,
|
|
"reward": -0.16875,
|
|
"reward_std": 0.5030868291854859,
|
|
"rewards/reward_fn/mean": -0.16875,
|
|
"rewards/reward_fn/std": 0.5030868351459503,
|
|
"sampling/importance_sampling_ratio/max": 1.266992461681366,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999652683734894,
|
|
"sampling/importance_sampling_ratio/min": 0.7717308163642883,
|
|
"sampling/sampling_logp_difference/max": 0.2738988041877747,
|
|
"sampling/sampling_logp_difference/mean": 0.021233023144304752,
|
|
"step": 6550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.7,
|
|
"completions/max_terminated_length": 289.7,
|
|
"completions/mean_length": 197.625,
|
|
"completions/mean_terminated_length": 197.625,
|
|
"completions/min_length": 129.4,
|
|
"completions/min_terminated_length": 129.4,
|
|
"entropy": 1.0125599063932895,
|
|
"epoch": 0.6031074744874506,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.874145030975342,
|
|
"kl": 0.05518628921126947,
|
|
"learning_rate": 4.694762953275554e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 28498717.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.3185652822256088,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.3185652881860733,
|
|
"sampling/importance_sampling_ratio/max": 1.2456906676292419,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002852916717528,
|
|
"sampling/importance_sampling_ratio/min": 0.7621093034744263,
|
|
"sampling/sampling_logp_difference/max": 0.2888748586177826,
|
|
"sampling/sampling_logp_difference/mean": 0.02088515181094408,
|
|
"step": 6560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 264.6,
|
|
"completions/max_terminated_length": 264.6,
|
|
"completions/mean_length": 190.5,
|
|
"completions/mean_terminated_length": 190.5,
|
|
"completions/min_length": 119.2,
|
|
"completions/min_terminated_length": 119.2,
|
|
"entropy": 1.0812375828623773,
|
|
"epoch": 0.6040268456375839,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.15348157286643982,
|
|
"kl": 0.054522328171879055,
|
|
"learning_rate": 4.6935486396734666e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 28542341.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.3193414121866226,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.3193414181470871,
|
|
"sampling/importance_sampling_ratio/max": 1.2910607695579528,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002699017524719,
|
|
"sampling/importance_sampling_ratio/min": 0.767870819568634,
|
|
"sampling/sampling_logp_difference/max": 0.3117747068405151,
|
|
"sampling/sampling_logp_difference/mean": 0.021191845275461675,
|
|
"step": 6570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.3,
|
|
"completions/max_terminated_length": 293.3,
|
|
"completions/mean_length": 204.025,
|
|
"completions/mean_terminated_length": 204.025,
|
|
"completions/min_length": 133.2,
|
|
"completions/min_terminated_length": 133.2,
|
|
"entropy": 1.1789320133626462,
|
|
"epoch": 0.6049462167877172,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 15.32407283782959,
|
|
"kl": 0.0939450132776983,
|
|
"learning_rate": 4.692332073121092e-06,
|
|
"loss": 0.0045,
|
|
"num_tokens": 28586503.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.3664708971977234,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.36647090315818787,
|
|
"sampling/importance_sampling_ratio/max": 1.2901464462280274,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993108451366425,
|
|
"sampling/importance_sampling_ratio/min": 0.7882357776165009,
|
|
"sampling/sampling_logp_difference/max": 0.28635857105255125,
|
|
"sampling/sampling_logp_difference/mean": 0.022379349917173386,
|
|
"step": 6580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 245.8,
|
|
"completions/max_terminated_length": 245.8,
|
|
"completions/mean_length": 183.625,
|
|
"completions/mean_terminated_length": 183.625,
|
|
"completions/min_length": 117.4,
|
|
"completions/min_terminated_length": 117.4,
|
|
"entropy": 0.9794266946613789,
|
|
"epoch": 0.6058655879378505,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.1575051099061966,
|
|
"kl": 0.030132370861247183,
|
|
"learning_rate": 4.691113254867942e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 28626881.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.1498008370399475,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.14980084896087648,
|
|
"sampling/importance_sampling_ratio/max": 1.2865644931793212,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994420886039734,
|
|
"sampling/importance_sampling_ratio/min": 0.7910012304782867,
|
|
"sampling/sampling_logp_difference/max": 0.266807496547699,
|
|
"sampling/sampling_logp_difference/mean": 0.020635079033672808,
|
|
"step": 6590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 251.9,
|
|
"completions/max_terminated_length": 251.9,
|
|
"completions/mean_length": 186.95,
|
|
"completions/mean_terminated_length": 186.95,
|
|
"completions/min_length": 111.1,
|
|
"completions/min_terminated_length": 111.1,
|
|
"entropy": 1.0408857516944408,
|
|
"epoch": 0.6067849590879838,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.724885940551758,
|
|
"kl": 0.05112304209615104,
|
|
"learning_rate": 4.68989218616584e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 28665701.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.32810779809951784,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.32810781002044676,
|
|
"sampling/importance_sampling_ratio/max": 1.2828383922576905,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003514289855957,
|
|
"sampling/importance_sampling_ratio/min": 0.7838038146495819,
|
|
"sampling/sampling_logp_difference/max": 0.2821976900100708,
|
|
"sampling/sampling_logp_difference/mean": 0.02131260298192501,
|
|
"step": 6600
|
|
},
|
|
{
|
|
"epoch": 0.6067849590879838,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0004363001745200698,
|
|
"eval_completions/max_length": 286.8394415357766,
|
|
"eval_completions/max_terminated_length": 286.5759162303665,
|
|
"eval_completions/mean_length": 199.5519197207679,
|
|
"eval_completions/mean_terminated_length": 199.44571181159162,
|
|
"eval_completions/min_length": 127.96335078534031,
|
|
"eval_completions/min_terminated_length": 127.96335078534031,
|
|
"eval_entropy": 1.0793265205715339,
|
|
"eval_frac_reward_zero_std": 0.30017452006980805,
|
|
"eval_kl": 0.048713106779872485,
|
|
"eval_loss": 0.002523747505620122,
|
|
"eval_num_tokens": 28665701.0,
|
|
"eval_reward": 0.0581369982547993,
|
|
"eval_reward_std": 0.3411068114310659,
|
|
"eval_rewards/reward_fn/mean": 0.0581369982547993,
|
|
"eval_rewards/reward_fn/std": 0.34110681964880923,
|
|
"eval_runtime": 1683.4142,
|
|
"eval_samples_per_second": 0.34,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2629436705957116,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999684097463132,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.774613283795628,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2772146638151239,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021778167501729093,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 6600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 250.2,
|
|
"completions/max_terminated_length": 250.2,
|
|
"completions/mean_length": 186.125,
|
|
"completions/mean_terminated_length": 186.125,
|
|
"completions/min_length": 125.9,
|
|
"completions/min_terminated_length": 125.9,
|
|
"entropy": 1.0622225984930993,
|
|
"epoch": 0.6077043302381171,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.780598163604736,
|
|
"kl": 0.05618832890177146,
|
|
"learning_rate": 4.688668868268921e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 28709863.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.3791164219379425,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.379116427898407,
|
|
"sampling/importance_sampling_ratio/max": 1.2290721774101256,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998210966587067,
|
|
"sampling/importance_sampling_ratio/min": 0.7747166216373443,
|
|
"sampling/sampling_logp_difference/max": 0.27154449820518495,
|
|
"sampling/sampling_logp_difference/mean": 0.020862479135394095,
|
|
"step": 6610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.0,
|
|
"completions/max_terminated_length": 297.0,
|
|
"completions/mean_length": 211.8875,
|
|
"completions/mean_terminated_length": 211.8875,
|
|
"completions/min_length": 125.9,
|
|
"completions/min_terminated_length": 125.9,
|
|
"entropy": 1.0691601939499378,
|
|
"epoch": 0.6086237013882504,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.949253082275391,
|
|
"kl": 0.042310426477342844,
|
|
"learning_rate": 4.687443302433631e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 28756574.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.3662448555231094,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.3662448674440384,
|
|
"sampling/importance_sampling_ratio/max": 1.249899423122406,
|
|
"sampling/importance_sampling_ratio/mean": 0.999539852142334,
|
|
"sampling/importance_sampling_ratio/min": 0.7916143417358399,
|
|
"sampling/sampling_logp_difference/max": 0.24048879146575927,
|
|
"sampling/sampling_logp_difference/mean": 0.02073957398533821,
|
|
"step": 6620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.9,
|
|
"completions/max_terminated_length": 280.9,
|
|
"completions/mean_length": 195.925,
|
|
"completions/mean_terminated_length": 195.925,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"entropy": 1.0465519703924655,
|
|
"epoch": 0.6095430725383837,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.619245529174805,
|
|
"kl": 0.036577882239362224,
|
|
"learning_rate": 4.686215489918724e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 28799064.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.45853824317455294,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.45853826105594636,
|
|
"sampling/importance_sampling_ratio/max": 1.2395642876625061,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000780403614045,
|
|
"sampling/importance_sampling_ratio/min": 0.7945998311042786,
|
|
"sampling/sampling_logp_difference/max": 0.2371358633041382,
|
|
"sampling/sampling_logp_difference/mean": 0.02094010226428509,
|
|
"step": 6630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.9,
|
|
"completions/max_terminated_length": 280.9,
|
|
"completions/mean_length": 198.6375,
|
|
"completions/mean_terminated_length": 198.6375,
|
|
"completions/min_length": 108.4,
|
|
"completions/min_terminated_length": 108.4,
|
|
"entropy": 1.0828476385679096,
|
|
"epoch": 0.6104624436885171,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.500866889953613,
|
|
"kl": 0.05637252437882125,
|
|
"learning_rate": 4.684985431985263e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 28840331.0,
|
|
"reward": 0.08125,
|
|
"reward_std": 0.4347601145505905,
|
|
"rewards/reward_fn/mean": 0.08125,
|
|
"rewards/reward_fn/std": 0.434760120511055,
|
|
"sampling/importance_sampling_ratio/max": 1.2804141521453858,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999092638492584,
|
|
"sampling/importance_sampling_ratio/min": 0.7911666989326477,
|
|
"sampling/sampling_logp_difference/max": 0.2663842558860779,
|
|
"sampling/sampling_logp_difference/mean": 0.021877037733793257,
|
|
"step": 6640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.7,
|
|
"completions/max_terminated_length": 273.7,
|
|
"completions/mean_length": 195.55,
|
|
"completions/mean_terminated_length": 195.55,
|
|
"completions/min_length": 122.1,
|
|
"completions/min_terminated_length": 122.1,
|
|
"entropy": 0.9815972581505775,
|
|
"epoch": 0.6113818148386504,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.434087753295898,
|
|
"kl": 0.04146459153853357,
|
|
"learning_rate": 4.6837531298966145e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 28882591.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.36193286776542666,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.36193287968635557,
|
|
"sampling/importance_sampling_ratio/max": 1.2819383144378662,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000441431999207,
|
|
"sampling/importance_sampling_ratio/min": 0.7762354671955108,
|
|
"sampling/sampling_logp_difference/max": 0.2784293174743652,
|
|
"sampling/sampling_logp_difference/mean": 0.020076168701052666,
|
|
"step": 6650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 321.1,
|
|
"completions/max_terminated_length": 321.1,
|
|
"completions/mean_length": 224.375,
|
|
"completions/mean_terminated_length": 224.375,
|
|
"completions/min_length": 136.3,
|
|
"completions/min_terminated_length": 136.3,
|
|
"entropy": 1.089103788137436,
|
|
"epoch": 0.6123011859887837,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.827288627624512,
|
|
"kl": 0.04946248646592721,
|
|
"learning_rate": 4.682518584918453e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 28927725.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.44064723551273344,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.4406472533941269,
|
|
"sampling/importance_sampling_ratio/max": 1.3398911118507386,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000384390354156,
|
|
"sampling/importance_sampling_ratio/min": 0.7956932306289672,
|
|
"sampling/sampling_logp_difference/max": 0.30133056640625,
|
|
"sampling/sampling_logp_difference/mean": 0.021049473993480207,
|
|
"step": 6660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.3,
|
|
"completions/max_terminated_length": 301.3,
|
|
"completions/mean_length": 203.8375,
|
|
"completions/mean_terminated_length": 203.8375,
|
|
"completions/min_length": 124.4,
|
|
"completions/min_terminated_length": 124.4,
|
|
"entropy": 1.090546839684248,
|
|
"epoch": 0.613220557138917,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.501370429992676,
|
|
"kl": 0.056024701485875995,
|
|
"learning_rate": 4.681281798318754e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 28973880.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.3636886715888977,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.3636886954307556,
|
|
"sampling/importance_sampling_ratio/max": 1.3274587988853455,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000105619430542,
|
|
"sampling/importance_sampling_ratio/min": 0.7645586431026459,
|
|
"sampling/sampling_logp_difference/max": 0.29791786670684817,
|
|
"sampling/sampling_logp_difference/mean": 0.02171063907444477,
|
|
"step": 6670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.9,
|
|
"completions/max_terminated_length": 300.9,
|
|
"completions/mean_length": 205.6625,
|
|
"completions/mean_terminated_length": 205.6625,
|
|
"completions/min_length": 129.8,
|
|
"completions/min_terminated_length": 129.8,
|
|
"entropy": 1.0911942847073077,
|
|
"epoch": 0.6141399282890503,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.215987205505371,
|
|
"kl": 0.0499262303346768,
|
|
"learning_rate": 4.680042771367797e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 29015477.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.27396377325057986,
|
|
"sampling/importance_sampling_ratio/max": 1.288773810863495,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000105381011963,
|
|
"sampling/importance_sampling_ratio/min": 0.7785434782505035,
|
|
"sampling/sampling_logp_difference/max": 0.2994304418563843,
|
|
"sampling/sampling_logp_difference/mean": 0.02125229239463806,
|
|
"step": 6680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.8,
|
|
"completions/max_terminated_length": 290.8,
|
|
"completions/mean_length": 212.175,
|
|
"completions/mean_terminated_length": 212.175,
|
|
"completions/min_length": 146.4,
|
|
"completions/min_terminated_length": 146.4,
|
|
"entropy": 1.1108071878552437,
|
|
"epoch": 0.6150592994391836,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.469494342803955,
|
|
"kl": 0.055718297022394836,
|
|
"learning_rate": 4.678801505338163e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 29061251.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.2276727557182312,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.22767276763916017,
|
|
"sampling/importance_sampling_ratio/max": 1.2935654878616334,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994342505931855,
|
|
"sampling/importance_sampling_ratio/min": 0.7713475346565246,
|
|
"sampling/sampling_logp_difference/max": 0.2980908274650574,
|
|
"sampling/sampling_logp_difference/mean": 0.021103793010115623,
|
|
"step": 6690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 318.5,
|
|
"completions/max_terminated_length": 318.5,
|
|
"completions/mean_length": 227.425,
|
|
"completions/mean_terminated_length": 227.425,
|
|
"completions/min_length": 137.6,
|
|
"completions/min_terminated_length": 137.6,
|
|
"entropy": 1.0685371793806553,
|
|
"epoch": 0.6159786705893169,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.516569137573242,
|
|
"kl": 0.05985648108180612,
|
|
"learning_rate": 4.67755800150473e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 29107701.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.39010659158229827,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.3901066094636917,
|
|
"sampling/importance_sampling_ratio/max": 1.3361659526824952,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999526917934418,
|
|
"sampling/importance_sampling_ratio/min": 0.7891739666461944,
|
|
"sampling/sampling_logp_difference/max": 0.29613181948661804,
|
|
"sampling/sampling_logp_difference/mean": 0.020833785459399224,
|
|
"step": 6700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.9,
|
|
"completions/max_terminated_length": 303.9,
|
|
"completions/mean_length": 211.8875,
|
|
"completions/mean_terminated_length": 211.8875,
|
|
"completions/min_length": 141.5,
|
|
"completions/min_terminated_length": 141.5,
|
|
"entropy": 1.0137500926852225,
|
|
"epoch": 0.6168980417394502,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.433523178100586,
|
|
"kl": 0.03887462035054341,
|
|
"learning_rate": 4.6763122611446765e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 29147516.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.4538504630327225,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.45385047495365144,
|
|
"sampling/importance_sampling_ratio/max": 1.285509753227234,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002160370349884,
|
|
"sampling/importance_sampling_ratio/min": 0.7662584841251373,
|
|
"sampling/sampling_logp_difference/max": 0.2902275621891022,
|
|
"sampling/sampling_logp_difference/mean": 0.020672991313040255,
|
|
"step": 6710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 320.9,
|
|
"completions/max_terminated_length": 320.9,
|
|
"completions/mean_length": 211.325,
|
|
"completions/mean_terminated_length": 211.325,
|
|
"completions/min_length": 132.1,
|
|
"completions/min_terminated_length": 132.1,
|
|
"entropy": 1.1308486185967923,
|
|
"epoch": 0.6178174128895835,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.557374000549316,
|
|
"kl": 0.059669853863306345,
|
|
"learning_rate": 4.675064285537479e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 29191206.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.4958795368671417,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.4958795428276062,
|
|
"sampling/importance_sampling_ratio/max": 1.4397983789443969,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003291189670562,
|
|
"sampling/importance_sampling_ratio/min": 0.7238905787467956,
|
|
"sampling/sampling_logp_difference/max": 0.4397250771522522,
|
|
"sampling/sampling_logp_difference/mean": 0.02223335150629282,
|
|
"step": 6720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.5,
|
|
"completions/max_terminated_length": 269.5,
|
|
"completions/mean_length": 197.6,
|
|
"completions/mean_terminated_length": 197.6,
|
|
"completions/min_length": 144.4,
|
|
"completions/min_terminated_length": 144.4,
|
|
"entropy": 1.011008794605732,
|
|
"epoch": 0.6187367840397169,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.798991680145264,
|
|
"kl": 0.08817185126245022,
|
|
"learning_rate": 4.6738140759649064e-06,
|
|
"loss": 0.0042,
|
|
"num_tokens": 29231878.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.35154260993003844,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.35154260993003844,
|
|
"sampling/importance_sampling_ratio/max": 1.3025209188461304,
|
|
"sampling/importance_sampling_ratio/mean": 1.000456589460373,
|
|
"sampling/importance_sampling_ratio/min": 0.7527783691883088,
|
|
"sampling/sampling_logp_difference/max": 0.3251086175441742,
|
|
"sampling/sampling_logp_difference/mean": 0.021010379120707513,
|
|
"step": 6730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.3,
|
|
"completions/max_terminated_length": 326.3,
|
|
"completions/mean_length": 235.35,
|
|
"completions/mean_terminated_length": 235.35,
|
|
"completions/min_length": 127.6,
|
|
"completions/min_terminated_length": 127.6,
|
|
"entropy": 1.0962674736976623,
|
|
"epoch": 0.6196561551898502,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.14123082160949707,
|
|
"kl": 0.04932711883448064,
|
|
"learning_rate": 4.672561633711026e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 29279858.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.2394672751426697,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.23946728110313414,
|
|
"sampling/importance_sampling_ratio/max": 1.3456030011177063,
|
|
"sampling/importance_sampling_ratio/mean": 1.000491976737976,
|
|
"sampling/importance_sampling_ratio/min": 0.8033839344978333,
|
|
"sampling/sampling_logp_difference/max": 0.2943825364112854,
|
|
"sampling/sampling_logp_difference/mean": 0.020889033563435078,
|
|
"step": 6740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.7,
|
|
"completions/max_terminated_length": 300.7,
|
|
"completions/mean_length": 220.0,
|
|
"completions/mean_terminated_length": 220.0,
|
|
"completions/min_length": 131.6,
|
|
"completions/min_terminated_length": 131.6,
|
|
"entropy": 1.067647210508585,
|
|
"epoch": 0.6205755263399835,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12703394889831543,
|
|
"kl": 0.048374779312871395,
|
|
"learning_rate": 4.671306960062195e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 29326538.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.3153284192085266,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.31532842516899107,
|
|
"sampling/importance_sampling_ratio/max": 1.2758236885070802,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001848340034485,
|
|
"sampling/importance_sampling_ratio/min": 0.7751625001430511,
|
|
"sampling/sampling_logp_difference/max": 0.27781530618667605,
|
|
"sampling/sampling_logp_difference/mean": 0.021103197894990443,
|
|
"step": 6750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 314.3,
|
|
"completions/max_terminated_length": 310.8,
|
|
"completions/mean_length": 220.2875,
|
|
"completions/mean_terminated_length": 216.39583435058594,
|
|
"completions/min_length": 127.9,
|
|
"completions/min_terminated_length": 127.9,
|
|
"entropy": 0.8945754617452621,
|
|
"epoch": 0.6214948974901168,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.18356207013130188,
|
|
"kl": 0.05154978273203596,
|
|
"learning_rate": 4.670050056307065e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 29368777.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.500145035982132,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.500145035982132,
|
|
"sampling/importance_sampling_ratio/max": 1.2708938002586365,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999270439147949,
|
|
"sampling/importance_sampling_ratio/min": 0.7740569829940795,
|
|
"sampling/sampling_logp_difference/max": 0.2744333386421204,
|
|
"sampling/sampling_logp_difference/mean": 0.018797744810581208,
|
|
"step": 6760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.5,
|
|
"completions/max_terminated_length": 305.5,
|
|
"completions/mean_length": 212.4625,
|
|
"completions/mean_terminated_length": 212.4625,
|
|
"completions/min_length": 138.3,
|
|
"completions/min_terminated_length": 138.3,
|
|
"entropy": 1.1364482261240483,
|
|
"epoch": 0.6224142686402501,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.612137794494629,
|
|
"kl": 0.0330553645035252,
|
|
"learning_rate": 4.668790923736577e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 29416086.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.45704681277275083,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.45704683661460876,
|
|
"sampling/importance_sampling_ratio/max": 1.274204635620117,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006697297096252,
|
|
"sampling/importance_sampling_ratio/min": 0.7953601777553558,
|
|
"sampling/sampling_logp_difference/max": 0.28102970123291016,
|
|
"sampling/sampling_logp_difference/mean": 0.021726584807038308,
|
|
"step": 6770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 312.7,
|
|
"completions/max_terminated_length": 298.2,
|
|
"completions/mean_length": 206.5625,
|
|
"completions/mean_terminated_length": 203.7482147216797,
|
|
"completions/min_length": 114.0,
|
|
"completions/min_terminated_length": 114.0,
|
|
"entropy": 1.0693196848034858,
|
|
"epoch": 0.6233336397903834,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.7688751220703125,
|
|
"kl": 0.04354674016358331,
|
|
"learning_rate": 4.667529563643959e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 29461411.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.4097943514585495,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.409794357419014,
|
|
"sampling/importance_sampling_ratio/max": 1.2784115552902222,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000968337059022,
|
|
"sampling/importance_sampling_ratio/min": 0.7785356223583222,
|
|
"sampling/sampling_logp_difference/max": 0.27309916019439695,
|
|
"sampling/sampling_logp_difference/mean": 0.021031028218567373,
|
|
"step": 6780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.4,
|
|
"completions/max_terminated_length": 300.4,
|
|
"completions/mean_length": 195.8625,
|
|
"completions/mean_terminated_length": 195.8625,
|
|
"completions/min_length": 120.0,
|
|
"completions/min_terminated_length": 120.0,
|
|
"entropy": 1.098496875911951,
|
|
"epoch": 0.6242530109405167,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12524226307868958,
|
|
"kl": 0.03582521768403239,
|
|
"learning_rate": 4.666265977324731e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 29502872.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.2777381867170334,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.27773819863796234,
|
|
"sampling/importance_sampling_ratio/max": 1.2434162735939025,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999323487281799,
|
|
"sampling/importance_sampling_ratio/min": 0.7974631726741791,
|
|
"sampling/sampling_logp_difference/max": 0.24550031423568724,
|
|
"sampling/sampling_logp_difference/mean": 0.021749641001224517,
|
|
"step": 6790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.1,
|
|
"completions/max_terminated_length": 290.1,
|
|
"completions/mean_length": 213.825,
|
|
"completions/mean_terminated_length": 213.825,
|
|
"completions/min_length": 131.5,
|
|
"completions/min_terminated_length": 131.5,
|
|
"entropy": 1.1424120552837849,
|
|
"epoch": 0.62517238209065,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.539258003234863,
|
|
"kl": 0.05649263090454042,
|
|
"learning_rate": 4.665000166076697e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 29545882.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.4628865152597427,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.4628865510225296,
|
|
"sampling/importance_sampling_ratio/max": 1.258629274368286,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995796322822571,
|
|
"sampling/importance_sampling_ratio/min": 0.7820207178592682,
|
|
"sampling/sampling_logp_difference/max": 0.2577647686004639,
|
|
"sampling/sampling_logp_difference/mean": 0.022256471402943134,
|
|
"step": 6800
|
|
},
|
|
{
|
|
"epoch": 0.62517238209065,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.001963350785340314,
|
|
"eval_completions/max_length": 307.2739965095986,
|
|
"eval_completions/max_terminated_length": 304.59685863874347,
|
|
"eval_completions/mean_length": 213.94349912739966,
|
|
"eval_completions/mean_terminated_length": 213.39765661216322,
|
|
"eval_completions/min_length": 137.63176265270505,
|
|
"eval_completions/min_terminated_length": 137.63176265270505,
|
|
"eval_entropy": 1.0944355514037047,
|
|
"eval_frac_reward_zero_std": 0.2670157068062827,
|
|
"eval_kl": 0.04171284452652848,
|
|
"eval_loss": 0.0021813286002725363,
|
|
"eval_num_tokens": 29545882.0,
|
|
"eval_reward": 0.06119109947643979,
|
|
"eval_reward_std": 0.3629122475522975,
|
|
"eval_rewards/reward_fn/mean": 0.06119109947643979,
|
|
"eval_rewards/reward_fn/std": 0.3629122584746146,
|
|
"eval_runtime": 1786.6055,
|
|
"eval_samples_per_second": 0.321,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.275020508241903,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000621857443406,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7714944661287737,
|
|
"eval_sampling/sampling_logp_difference/max": 0.28686973405758126,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021319506369713193,
|
|
"eval_steps_per_second": 0.04,
|
|
"step": 6800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.0,
|
|
"completions/max_terminated_length": 324.0,
|
|
"completions/mean_length": 215.1,
|
|
"completions/mean_terminated_length": 215.1,
|
|
"completions/min_length": 136.3,
|
|
"completions/min_terminated_length": 136.3,
|
|
"entropy": 1.1555157959461213,
|
|
"epoch": 0.6260917532407833,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.354942321777344,
|
|
"kl": 0.04119767733500339,
|
|
"learning_rate": 4.663732131199946e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 29590578.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.4903250575065613,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.4903250753879547,
|
|
"sampling/importance_sampling_ratio/max": 1.243701958656311,
|
|
"sampling/importance_sampling_ratio/mean": 0.999388587474823,
|
|
"sampling/importance_sampling_ratio/min": 0.772141021490097,
|
|
"sampling/sampling_logp_difference/max": 0.26975553035736083,
|
|
"sampling/sampling_logp_difference/mean": 0.022179535403847695,
|
|
"step": 6810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 314.5,
|
|
"completions/max_terminated_length": 287.4,
|
|
"completions/mean_length": 211.85,
|
|
"completions/mean_terminated_length": 207.81964416503905,
|
|
"completions/min_length": 137.9,
|
|
"completions/min_terminated_length": 137.9,
|
|
"entropy": 1.1244710437953471,
|
|
"epoch": 0.6270111243909167,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.13574959337711334,
|
|
"kl": 0.04340720770414919,
|
|
"learning_rate": 4.662461873996853e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 29633278.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.35747711062431337,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.35747711062431337,
|
|
"sampling/importance_sampling_ratio/max": 1.3211257815361024,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002057731151581,
|
|
"sampling/importance_sampling_ratio/min": 0.7779735743999481,
|
|
"sampling/sampling_logp_difference/max": 0.2981942296028137,
|
|
"sampling/sampling_logp_difference/mean": 0.021695419773459436,
|
|
"step": 6820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.2,
|
|
"completions/max_terminated_length": 283.2,
|
|
"completions/mean_length": 205.275,
|
|
"completions/mean_terminated_length": 205.275,
|
|
"completions/min_length": 140.7,
|
|
"completions/min_terminated_length": 140.7,
|
|
"entropy": 1.1168083816766738,
|
|
"epoch": 0.62793049554105,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.14704281091690063,
|
|
"kl": 0.04773867194307968,
|
|
"learning_rate": 4.6611893957720715e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 29681700.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.3530599385499954,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.3530599445104599,
|
|
"sampling/importance_sampling_ratio/max": 1.2660350322723388,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005456387996674,
|
|
"sampling/importance_sampling_ratio/min": 0.771111911535263,
|
|
"sampling/sampling_logp_difference/max": 0.2757743239402771,
|
|
"sampling/sampling_logp_difference/mean": 0.02183311153203249,
|
|
"step": 6830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.4,
|
|
"completions/max_terminated_length": 278.4,
|
|
"completions/mean_length": 208.9625,
|
|
"completions/mean_terminated_length": 208.9625,
|
|
"completions/min_length": 134.3,
|
|
"completions/min_terminated_length": 134.3,
|
|
"entropy": 1.0161209784448146,
|
|
"epoch": 0.6288498666911833,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.436909198760986,
|
|
"kl": 0.06230252794921398,
|
|
"learning_rate": 4.6599146978325405e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 29726153.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.4254619300365448,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.42546194791793823,
|
|
"sampling/importance_sampling_ratio/max": 1.2658935189247131,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994531333446502,
|
|
"sampling/importance_sampling_ratio/min": 0.7994739711284637,
|
|
"sampling/sampling_logp_difference/max": 0.25241904258728026,
|
|
"sampling/sampling_logp_difference/mean": 0.019927116483449935,
|
|
"step": 6840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.7,
|
|
"completions/max_terminated_length": 306.7,
|
|
"completions/mean_length": 217.425,
|
|
"completions/mean_terminated_length": 217.425,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 1.1072007894515992,
|
|
"epoch": 0.6297692378413166,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14334885776042938,
|
|
"kl": 0.055353190226014704,
|
|
"learning_rate": 4.658637781487477e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 29769859.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.3466688096523285,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.3466688334941864,
|
|
"sampling/importance_sampling_ratio/max": 1.2791266441345215,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000653445720673,
|
|
"sampling/importance_sampling_ratio/min": 0.7633672654628754,
|
|
"sampling/sampling_logp_difference/max": 0.29541170597076416,
|
|
"sampling/sampling_logp_difference/mean": 0.021611334197223185,
|
|
"step": 6850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 323.6,
|
|
"completions/max_terminated_length": 300.3,
|
|
"completions/mean_length": 208.8125,
|
|
"completions/mean_terminated_length": 205.14464416503907,
|
|
"completions/min_length": 127.9,
|
|
"completions/min_terminated_length": 127.9,
|
|
"entropy": 1.1705016158521175,
|
|
"epoch": 0.6306886089914499,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13951478898525238,
|
|
"kl": 0.0459657346829772,
|
|
"learning_rate": 4.657358648048375e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 29812716.0,
|
|
"reward": -0.00625,
|
|
"reward_std": 0.2870673507452011,
|
|
"rewards/reward_fn/mean": -0.00625,
|
|
"rewards/reward_fn/std": 0.2870673507452011,
|
|
"sampling/importance_sampling_ratio/max": 1.2458139300346374,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998774826526642,
|
|
"sampling/importance_sampling_ratio/min": 0.7827580690383911,
|
|
"sampling/sampling_logp_difference/max": 0.25504260063171386,
|
|
"sampling/sampling_logp_difference/mean": 0.02203933708369732,
|
|
"step": 6860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.9,
|
|
"completions/max_terminated_length": 290.9,
|
|
"completions/mean_length": 198.0,
|
|
"completions/mean_terminated_length": 198.0,
|
|
"completions/min_length": 121.5,
|
|
"completions/min_terminated_length": 121.5,
|
|
"entropy": 1.0373144365847111,
|
|
"epoch": 0.6316079801415831,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.13260096311569214,
|
|
"kl": 0.03887657629675232,
|
|
"learning_rate": 4.656077298829009e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 29852348.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.2675303816795349,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.26753038763999937,
|
|
"sampling/importance_sampling_ratio/max": 1.2509888768196107,
|
|
"sampling/importance_sampling_ratio/mean": 1.000206995010376,
|
|
"sampling/importance_sampling_ratio/min": 0.7825287640094757,
|
|
"sampling/sampling_logp_difference/max": 0.2552851915359497,
|
|
"sampling/sampling_logp_difference/mean": 0.02099158223718405,
|
|
"step": 6870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 262.0,
|
|
"completions/max_terminated_length": 262.0,
|
|
"completions/mean_length": 194.6625,
|
|
"completions/mean_terminated_length": 194.6625,
|
|
"completions/min_length": 131.9,
|
|
"completions/min_terminated_length": 131.9,
|
|
"entropy": 1.127125496417284,
|
|
"epoch": 0.6325273512917164,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.785040378570557,
|
|
"kl": 0.04155215995851904,
|
|
"learning_rate": 4.654793735145425e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 29896321.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.38308919668197633,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.3830892026424408,
|
|
"sampling/importance_sampling_ratio/max": 1.3237892270088196,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999879240989685,
|
|
"sampling/importance_sampling_ratio/min": 0.7832675039768219,
|
|
"sampling/sampling_logp_difference/max": 0.30809980630874634,
|
|
"sampling/sampling_logp_difference/mean": 0.022603087686002256,
|
|
"step": 6880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 323.3,
|
|
"completions/max_terminated_length": 319.0,
|
|
"completions/mean_length": 225.7,
|
|
"completions/mean_terminated_length": 220.46666870117187,
|
|
"completions/min_length": 140.5,
|
|
"completions/min_terminated_length": 140.5,
|
|
"entropy": 1.1387503877282144,
|
|
"epoch": 0.6334467224418497,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 8.038926124572754,
|
|
"kl": 0.044994328450411555,
|
|
"learning_rate": 4.653507958315948e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 29941993.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.40024718046188357,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.40024718046188357,
|
|
"sampling/importance_sampling_ratio/max": 1.2955519795417785,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002415895462036,
|
|
"sampling/importance_sampling_ratio/min": 0.7670924007892609,
|
|
"sampling/sampling_logp_difference/max": 0.29513548612594603,
|
|
"sampling/sampling_logp_difference/mean": 0.022347428277134894,
|
|
"step": 6890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.9,
|
|
"completions/max_terminated_length": 273.9,
|
|
"completions/mean_length": 195.175,
|
|
"completions/mean_terminated_length": 195.175,
|
|
"completions/min_length": 121.7,
|
|
"completions/min_terminated_length": 121.7,
|
|
"entropy": 1.0728932473808528,
|
|
"epoch": 0.634366093591983,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.686387538909912,
|
|
"kl": 0.0415146428742446,
|
|
"learning_rate": 4.652219969661174e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 29985351.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.18685350716114044,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.18685351312160492,
|
|
"sampling/importance_sampling_ratio/max": 1.2598000288009643,
|
|
"sampling/importance_sampling_ratio/mean": 1.000317108631134,
|
|
"sampling/importance_sampling_ratio/min": 0.7552180290222168,
|
|
"sampling/sampling_logp_difference/max": 0.3066049098968506,
|
|
"sampling/sampling_logp_difference/mean": 0.021217455342411994,
|
|
"step": 6900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.7,
|
|
"completions/max_terminated_length": 288.7,
|
|
"completions/mean_length": 202.825,
|
|
"completions/mean_terminated_length": 202.825,
|
|
"completions/min_length": 139.6,
|
|
"completions/min_terminated_length": 139.6,
|
|
"entropy": 1.0837183758616447,
|
|
"epoch": 0.6352854647421164,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.21113154292106628,
|
|
"kl": 0.04661462446092628,
|
|
"learning_rate": 4.650929770503968e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 30030049.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.4493387460708618,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.44933876395225525,
|
|
"sampling/importance_sampling_ratio/max": 1.3036141753196717,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001380920410157,
|
|
"sampling/importance_sampling_ratio/min": 0.7622316956520081,
|
|
"sampling/sampling_logp_difference/max": 0.30969287157058717,
|
|
"sampling/sampling_logp_difference/mean": 0.021134368795901536,
|
|
"step": 6910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 325.1,
|
|
"completions/max_terminated_length": 307.9,
|
|
"completions/mean_length": 227.325,
|
|
"completions/mean_terminated_length": 220.90178833007812,
|
|
"completions/min_length": 154.3,
|
|
"completions/min_terminated_length": 154.3,
|
|
"entropy": 1.0782302975654603,
|
|
"epoch": 0.6362048358922497,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.446222305297852,
|
|
"kl": 0.04903796419966966,
|
|
"learning_rate": 4.64963736216947e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 30075451.0,
|
|
"reward": -0.10625,
|
|
"reward_std": 0.31606110036373136,
|
|
"rewards/reward_fn/mean": -0.10625,
|
|
"rewards/reward_fn/std": 0.31606111228466033,
|
|
"sampling/importance_sampling_ratio/max": 1.2200359940528869,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996735095977783,
|
|
"sampling/importance_sampling_ratio/min": 0.7796869039535522,
|
|
"sampling/sampling_logp_difference/max": 0.24958672523498535,
|
|
"sampling/sampling_logp_difference/mean": 0.021077096182852983,
|
|
"step": 6920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.5,
|
|
"completions/max_terminated_length": 300.5,
|
|
"completions/mean_length": 198.1875,
|
|
"completions/mean_terminated_length": 198.1875,
|
|
"completions/min_length": 108.2,
|
|
"completions/min_terminated_length": 108.2,
|
|
"entropy": 1.0618108380585909,
|
|
"epoch": 0.637124207042383,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.563666820526123,
|
|
"kl": 0.03885188545100391,
|
|
"learning_rate": 4.648342745985087e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 30121498.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.3978420317173004,
|
|
"rewards/reward_fn/mean": 0.425,
|
|
"rewards/reward_fn/std": 0.3978420555591583,
|
|
"sampling/importance_sampling_ratio/max": 1.2341456890106202,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005105316638947,
|
|
"sampling/importance_sampling_ratio/min": 0.7986541211605072,
|
|
"sampling/sampling_logp_difference/max": 0.24060747623443604,
|
|
"sampling/sampling_logp_difference/mean": 0.02072082869708538,
|
|
"step": 6930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.9,
|
|
"completions/max_terminated_length": 291.9,
|
|
"completions/mean_length": 201.175,
|
|
"completions/mean_terminated_length": 201.175,
|
|
"completions/min_length": 116.7,
|
|
"completions/min_terminated_length": 116.7,
|
|
"entropy": 1.1433960556983949,
|
|
"epoch": 0.6380435781925163,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.469820976257324,
|
|
"kl": 0.04899288679007441,
|
|
"learning_rate": 4.647045923280493e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 30164136.0,
|
|
"reward": -0.03125,
|
|
"reward_std": 0.5548639625310898,
|
|
"rewards/reward_fn/mean": -0.03125,
|
|
"rewards/reward_fn/std": 0.5548639863729476,
|
|
"sampling/importance_sampling_ratio/max": 1.2946020126342774,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998938620090485,
|
|
"sampling/importance_sampling_ratio/min": 0.771307361125946,
|
|
"sampling/sampling_logp_difference/max": 0.2983285427093506,
|
|
"sampling/sampling_logp_difference/mean": 0.02288770768791437,
|
|
"step": 6940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 256.1,
|
|
"completions/max_terminated_length": 256.1,
|
|
"completions/mean_length": 182.5625,
|
|
"completions/mean_terminated_length": 182.5625,
|
|
"completions/min_length": 121.5,
|
|
"completions/min_terminated_length": 121.5,
|
|
"entropy": 1.0454321503639221,
|
|
"epoch": 0.6389629493426496,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.0962419509887695,
|
|
"kl": 0.03612101135076955,
|
|
"learning_rate": 4.645746895387628e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 30203525.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.39266277551651,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.3926627814769745,
|
|
"sampling/importance_sampling_ratio/max": 1.3027013540267944,
|
|
"sampling/importance_sampling_ratio/mean": 1.000362628698349,
|
|
"sampling/importance_sampling_ratio/min": 0.7952675461769104,
|
|
"sampling/sampling_logp_difference/max": 0.26876828670501707,
|
|
"sampling/sampling_logp_difference/mean": 0.02092335410416126,
|
|
"step": 6950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.4,
|
|
"completions/max_terminated_length": 309.4,
|
|
"completions/mean_length": 196.6625,
|
|
"completions/mean_terminated_length": 196.6625,
|
|
"completions/min_length": 116.4,
|
|
"completions/min_terminated_length": 116.4,
|
|
"entropy": 1.0862117640674114,
|
|
"epoch": 0.6398823204927829,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.281369209289551,
|
|
"kl": 0.046541214553872126,
|
|
"learning_rate": 4.6444456636407e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 30244858.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.36193286776542666,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.36193287968635557,
|
|
"sampling/importance_sampling_ratio/max": 1.2642287731170654,
|
|
"sampling/importance_sampling_ratio/mean": 0.999469417333603,
|
|
"sampling/importance_sampling_ratio/min": 0.7357144683599472,
|
|
"sampling/sampling_logp_difference/max": 0.32335125207901,
|
|
"sampling/sampling_logp_difference/mean": 0.020689988601952793,
|
|
"step": 6960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.7,
|
|
"completions/max_terminated_length": 308.7,
|
|
"completions/mean_length": 193.1875,
|
|
"completions/mean_terminated_length": 193.1875,
|
|
"completions/min_length": 132.3,
|
|
"completions/min_terminated_length": 132.3,
|
|
"entropy": 1.160147128254175,
|
|
"epoch": 0.6408016916429162,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.213428497314453,
|
|
"kl": 0.06023726573912427,
|
|
"learning_rate": 4.643142229376178e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 30289065.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.3438155859708786,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.34381560385227206,
|
|
"sampling/importance_sampling_ratio/max": 1.241119384765625,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001249015331268,
|
|
"sampling/importance_sampling_ratio/min": 0.7904399693012237,
|
|
"sampling/sampling_logp_difference/max": 0.24830453395843505,
|
|
"sampling/sampling_logp_difference/mean": 0.022625867277383804,
|
|
"step": 6970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 257.4,
|
|
"completions/max_terminated_length": 257.4,
|
|
"completions/mean_length": 186.9375,
|
|
"completions/mean_terminated_length": 186.9375,
|
|
"completions/min_length": 116.6,
|
|
"completions/min_terminated_length": 116.6,
|
|
"entropy": 1.1095874726772308,
|
|
"epoch": 0.6417210627930495,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.596249103546143,
|
|
"kl": 0.037959228648105636,
|
|
"learning_rate": 4.64183659393279e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 30327364.0,
|
|
"reward": -0.2,
|
|
"reward_std": 0.5650873184204102,
|
|
"rewards/reward_fn/mean": -0.2,
|
|
"rewards/reward_fn/std": 0.5650873243808746,
|
|
"sampling/importance_sampling_ratio/max": 1.2830547928810119,
|
|
"sampling/importance_sampling_ratio/mean": 1.00035280585289,
|
|
"sampling/importance_sampling_ratio/min": 0.7854923188686371,
|
|
"sampling/sampling_logp_difference/max": 0.26614352464675906,
|
|
"sampling/sampling_logp_difference/mean": 0.022300955839455128,
|
|
"step": 6980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.9,
|
|
"completions/max_terminated_length": 283.9,
|
|
"completions/mean_length": 202.4625,
|
|
"completions/mean_terminated_length": 202.4625,
|
|
"completions/min_length": 132.5,
|
|
"completions/min_terminated_length": 132.5,
|
|
"entropy": 1.1392049722373485,
|
|
"epoch": 0.6426404339431828,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.119923114776611,
|
|
"kl": 0.04383351393043995,
|
|
"learning_rate": 4.640528758651532e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 30369929.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.5186047405004501,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.5186047583818436,
|
|
"sampling/importance_sampling_ratio/max": 1.342177724838257,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002257466316222,
|
|
"sampling/importance_sampling_ratio/min": 0.7912344515323639,
|
|
"sampling/sampling_logp_difference/max": 0.30578161478042604,
|
|
"sampling/sampling_logp_difference/mean": 0.021541558764874937,
|
|
"step": 6990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 331.3,
|
|
"completions/max_terminated_length": 324.0,
|
|
"completions/mean_length": 225.4875,
|
|
"completions/mean_terminated_length": 222.7125,
|
|
"completions/min_length": 129.1,
|
|
"completions/min_terminated_length": 129.1,
|
|
"entropy": 1.0147764183580876,
|
|
"epoch": 0.6435598050933161,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.35733526945114136,
|
|
"kl": 0.03853838726645335,
|
|
"learning_rate": 4.639218724875655e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 30410288.0,
|
|
"reward": -0.05625,
|
|
"reward_std": 0.48227610886096955,
|
|
"rewards/reward_fn/mean": -0.05625,
|
|
"rewards/reward_fn/std": 0.48227610886096955,
|
|
"sampling/importance_sampling_ratio/max": 1.3069356441497804,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995644807815551,
|
|
"sampling/importance_sampling_ratio/min": 0.7649729728698731,
|
|
"sampling/sampling_logp_difference/max": 0.31061495542526246,
|
|
"sampling/sampling_logp_difference/mean": 0.02032538391649723,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"epoch": 0.6435598050933161,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0015270506108202443,
|
|
"eval_completions/max_length": 299.3752181500873,
|
|
"eval_completions/max_terminated_length": 297.4851657940663,
|
|
"eval_completions/mean_length": 207.52028795811518,
|
|
"eval_completions/mean_terminated_length": 207.1254987999615,
|
|
"eval_completions/min_length": 132.5043630017452,
|
|
"eval_completions/min_terminated_length": 132.5043630017452,
|
|
"eval_entropy": 1.1399416261422488,
|
|
"eval_frac_reward_zero_std": 0.2600349040139616,
|
|
"eval_kl": 0.04126270575835703,
|
|
"eval_loss": 0.002148049185052514,
|
|
"eval_num_tokens": 30410288.0,
|
|
"eval_reward": 0.03588568935427574,
|
|
"eval_reward_std": 0.3895306445229116,
|
|
"eval_rewards/reward_fn/mean": 0.03588568935427574,
|
|
"eval_rewards/reward_fn/std": 0.38953065565327283,
|
|
"eval_runtime": 1744.9092,
|
|
"eval_samples_per_second": 0.328,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2754711324631856,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.000052079584378,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7804716802080264,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27688996413199274,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021683384202758782,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 361.9,
|
|
"completions/max_terminated_length": 361.9,
|
|
"completions/mean_length": 234.0875,
|
|
"completions/mean_terminated_length": 234.0875,
|
|
"completions/min_length": 157.4,
|
|
"completions/min_terminated_length": 157.4,
|
|
"entropy": 1.1878170371055603,
|
|
"epoch": 0.6444791762434495,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.17004451155662537,
|
|
"kl": 0.04644565583439544,
|
|
"learning_rate": 4.637906493950668e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 30455367.0,
|
|
"reward": -0.2625,
|
|
"reward_std": 0.353536981344223,
|
|
"rewards/reward_fn/mean": -0.2625,
|
|
"rewards/reward_fn/std": 0.353536993265152,
|
|
"sampling/importance_sampling_ratio/max": 1.2563496351242065,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998129367828369,
|
|
"sampling/importance_sampling_ratio/min": 0.7519959330558776,
|
|
"sampling/sampling_logp_difference/max": 0.3017697691917419,
|
|
"sampling/sampling_logp_difference/mean": 0.022627398557960986,
|
|
"step": 7010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.7,
|
|
"completions/max_terminated_length": 277.7,
|
|
"completions/mean_length": 214.725,
|
|
"completions/mean_terminated_length": 214.725,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 1.1484743781387805,
|
|
"epoch": 0.6453985473935828,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1319764107465744,
|
|
"kl": 0.03933565615443513,
|
|
"learning_rate": 4.636592067224337e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 30499057.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.3130935251712799,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.3130935370922089,
|
|
"sampling/importance_sampling_ratio/max": 1.261081600189209,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996011137962342,
|
|
"sampling/importance_sampling_ratio/min": 0.7873663008213043,
|
|
"sampling/sampling_logp_difference/max": 0.25997090339660645,
|
|
"sampling/sampling_logp_difference/mean": 0.022218709625303746,
|
|
"step": 7020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 335.9,
|
|
"completions/max_terminated_length": 335.9,
|
|
"completions/mean_length": 227.275,
|
|
"completions/mean_terminated_length": 227.275,
|
|
"completions/min_length": 149.6,
|
|
"completions/min_terminated_length": 149.6,
|
|
"entropy": 1.2338936746120452,
|
|
"epoch": 0.6463179185437161,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.18731720745563507,
|
|
"kl": 0.05541989308549091,
|
|
"learning_rate": 4.635275446046685e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 30545191.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.332049286365509,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.3320492923259735,
|
|
"sampling/importance_sampling_ratio/max": 1.31266348361969,
|
|
"sampling/importance_sampling_ratio/mean": 0.999973726272583,
|
|
"sampling/importance_sampling_ratio/min": 0.7816827774047852,
|
|
"sampling/sampling_logp_difference/max": 0.29988530874252317,
|
|
"sampling/sampling_logp_difference/mean": 0.022703452967107296,
|
|
"step": 7030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 305.7,
|
|
"completions/max_terminated_length": 305.7,
|
|
"completions/mean_length": 222.2875,
|
|
"completions/mean_terminated_length": 222.2875,
|
|
"completions/min_length": 150.7,
|
|
"completions/min_terminated_length": 150.7,
|
|
"entropy": 1.1482506163418293,
|
|
"epoch": 0.6472372896938494,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.908675670623779,
|
|
"kl": 0.04748713891021907,
|
|
"learning_rate": 4.633956631769986e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 30591022.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.4153409838676453,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.41534098982810974,
|
|
"sampling/importance_sampling_ratio/max": 1.2869335412979126,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994430124759675,
|
|
"sampling/importance_sampling_ratio/min": 0.7627106249332428,
|
|
"sampling/sampling_logp_difference/max": 0.31466342210769654,
|
|
"sampling/sampling_logp_difference/mean": 0.02198565937578678,
|
|
"step": 7040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.0,
|
|
"completions/max_terminated_length": 306.0,
|
|
"completions/mean_length": 216.75,
|
|
"completions/mean_terminated_length": 216.75,
|
|
"completions/min_length": 150.5,
|
|
"completions/min_terminated_length": 150.5,
|
|
"entropy": 1.1612841099500657,
|
|
"epoch": 0.6481566608439827,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.825497627258301,
|
|
"kl": 0.05818576728925109,
|
|
"learning_rate": 4.632635625748768e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 30632786.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.33751319646835326,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.3375132083892822,
|
|
"sampling/importance_sampling_ratio/max": 1.3121665120124817,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999531984329224,
|
|
"sampling/importance_sampling_ratio/min": 0.7557978451251983,
|
|
"sampling/sampling_logp_difference/max": 0.32109909057617186,
|
|
"sampling/sampling_logp_difference/mean": 0.02272858116775751,
|
|
"step": 7050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.3,
|
|
"completions/max_terminated_length": 306.3,
|
|
"completions/mean_length": 208.4625,
|
|
"completions/mean_terminated_length": 208.4625,
|
|
"completions/min_length": 125.6,
|
|
"completions/min_terminated_length": 125.6,
|
|
"entropy": 1.0524587467312814,
|
|
"epoch": 0.649076031994116,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.919009208679199,
|
|
"kl": 0.0453869518940337,
|
|
"learning_rate": 4.631312429339811e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 30674471.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.25587469935417173,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.25587469935417173,
|
|
"sampling/importance_sampling_ratio/max": 1.2805373191833496,
|
|
"sampling/importance_sampling_ratio/mean": 1.00029855966568,
|
|
"sampling/importance_sampling_ratio/min": 0.7590613961219788,
|
|
"sampling/sampling_logp_difference/max": 0.3037511229515076,
|
|
"sampling/sampling_logp_difference/mean": 0.021444643288850783,
|
|
"step": 7060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 318.2,
|
|
"completions/max_terminated_length": 295.5,
|
|
"completions/mean_length": 226.675,
|
|
"completions/mean_terminated_length": 223.1125015258789,
|
|
"completions/min_length": 147.9,
|
|
"completions/min_terminated_length": 147.9,
|
|
"entropy": 1.148937001824379,
|
|
"epoch": 0.6499954031442493,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.626952648162842,
|
|
"kl": 0.056578563433140515,
|
|
"learning_rate": 4.6299870439021446e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 30719565.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.33991776406764984,
|
|
"rewards/reward_fn/mean": 0.03125,
|
|
"rewards/reward_fn/std": 0.33991776406764984,
|
|
"sampling/importance_sampling_ratio/max": 1.2652281999588013,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006783246994018,
|
|
"sampling/importance_sampling_ratio/min": 0.7524769246578217,
|
|
"sampling/sampling_logp_difference/max": 0.2974590539932251,
|
|
"sampling/sampling_logp_difference/mean": 0.02250605672597885,
|
|
"step": 7070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.9,
|
|
"completions/max_terminated_length": 280.9,
|
|
"completions/mean_length": 210.5875,
|
|
"completions/mean_terminated_length": 210.5875,
|
|
"completions/min_length": 153.9,
|
|
"completions/min_terminated_length": 153.9,
|
|
"entropy": 1.0896439947187901,
|
|
"epoch": 0.6509147742943826,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 7.971179485321045,
|
|
"kl": 0.05408676060615107,
|
|
"learning_rate": 4.628659470797044e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 30762316.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.2270910620689392,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.22709106802940368,
|
|
"sampling/importance_sampling_ratio/max": 1.2525829911231994,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000336110591888,
|
|
"sampling/importance_sampling_ratio/min": 0.7763023734092712,
|
|
"sampling/sampling_logp_difference/max": 0.26648181676864624,
|
|
"sampling/sampling_logp_difference/mean": 0.022121440246701242,
|
|
"step": 7080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.2,
|
|
"completions/max_terminated_length": 307.2,
|
|
"completions/mean_length": 213.75,
|
|
"completions/mean_terminated_length": 213.75,
|
|
"completions/min_length": 138.4,
|
|
"completions/min_terminated_length": 138.4,
|
|
"entropy": 1.1037369690835477,
|
|
"epoch": 0.6518341454445159,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.10703787207603455,
|
|
"kl": 0.04414446228765882,
|
|
"learning_rate": 4.627329711388033e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 30807432.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.261338609457016,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.2613386154174805,
|
|
"sampling/importance_sampling_ratio/max": 1.2566338777542114,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004343688488007,
|
|
"sampling/importance_sampling_ratio/min": 0.7955641984939575,
|
|
"sampling/sampling_logp_difference/max": 0.2435443878173828,
|
|
"sampling/sampling_logp_difference/mean": 0.021285582892596723,
|
|
"step": 7090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 267.1,
|
|
"completions/max_terminated_length": 267.1,
|
|
"completions/mean_length": 194.5125,
|
|
"completions/mean_terminated_length": 194.5125,
|
|
"completions/min_length": 121.2,
|
|
"completions/min_terminated_length": 121.2,
|
|
"entropy": 1.1432061351835727,
|
|
"epoch": 0.6527535165946493,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.29132604598999,
|
|
"kl": 0.05914498777128756,
|
|
"learning_rate": 4.625997767040883e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 30846545.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.5240216553211212,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.5240216672420501,
|
|
"sampling/importance_sampling_ratio/max": 1.2507328391075134,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995620131492615,
|
|
"sampling/importance_sampling_ratio/min": 0.7924042165279388,
|
|
"sampling/sampling_logp_difference/max": 0.248442405462265,
|
|
"sampling/sampling_logp_difference/mean": 0.02318373918533325,
|
|
"step": 7100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.1,
|
|
"completions/max_terminated_length": 316.1,
|
|
"completions/mean_length": 211.9,
|
|
"completions/mean_terminated_length": 211.9,
|
|
"completions/min_length": 142.7,
|
|
"completions/min_terminated_length": 142.7,
|
|
"entropy": 1.1003998763859273,
|
|
"epoch": 0.6536728877447826,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.16145722568035126,
|
|
"kl": 0.04716645225416869,
|
|
"learning_rate": 4.624663639123607e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 30892137.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.27549403011798856,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.27549404203891753,
|
|
"sampling/importance_sampling_ratio/max": 1.2743995308876037,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994609892368317,
|
|
"sampling/importance_sampling_ratio/min": 0.7787565529346466,
|
|
"sampling/sampling_logp_difference/max": 0.28251625299453736,
|
|
"sampling/sampling_logp_difference/mean": 0.02230153065174818,
|
|
"step": 7110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.9,
|
|
"completions/max_terminated_length": 303.9,
|
|
"completions/mean_length": 219.0625,
|
|
"completions/mean_terminated_length": 219.0625,
|
|
"completions/min_length": 153.7,
|
|
"completions/min_terminated_length": 153.7,
|
|
"entropy": 1.115867978334427,
|
|
"epoch": 0.6545922588949159,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.820727825164795,
|
|
"kl": 0.05551559854648076,
|
|
"learning_rate": 4.623327329006461e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 30935966.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.2504029482603073,
|
|
"sampling/importance_sampling_ratio/max": 1.2652942061424255,
|
|
"sampling/importance_sampling_ratio/mean": 1.000111174583435,
|
|
"sampling/importance_sampling_ratio/min": 0.7491200685501098,
|
|
"sampling/sampling_logp_difference/max": 0.2989262342453003,
|
|
"sampling/sampling_logp_difference/mean": 0.021689656376838683,
|
|
"step": 7120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.3,
|
|
"completions/max_terminated_length": 304.3,
|
|
"completions/mean_length": 214.1,
|
|
"completions/mean_terminated_length": 214.1,
|
|
"completions/min_length": 145.0,
|
|
"completions/min_terminated_length": 145.0,
|
|
"entropy": 1.1453317418694495,
|
|
"epoch": 0.6555116300450492,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.079078674316406,
|
|
"kl": 0.05883527337573469,
|
|
"learning_rate": 4.621988838061942e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 30978470.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.3423954129219055,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.3423954248428345,
|
|
"sampling/importance_sampling_ratio/max": 1.2732753038406373,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997225821018219,
|
|
"sampling/importance_sampling_ratio/min": 0.7677494645118713,
|
|
"sampling/sampling_logp_difference/max": 0.28570563197135923,
|
|
"sampling/sampling_logp_difference/mean": 0.021984077244997024,
|
|
"step": 7130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.2,
|
|
"completions/max_terminated_length": 304.2,
|
|
"completions/mean_length": 219.525,
|
|
"completions/mean_terminated_length": 219.525,
|
|
"completions/min_length": 140.4,
|
|
"completions/min_terminated_length": 140.4,
|
|
"entropy": 1.0643950961530209,
|
|
"epoch": 0.6564310011951825,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.963262557983398,
|
|
"kl": 0.045676520210690796,
|
|
"learning_rate": 4.620648167664789e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 31020784.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.3288124233484268,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.3288124293088913,
|
|
"sampling/importance_sampling_ratio/max": 1.2813105940818788,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999165058135986,
|
|
"sampling/importance_sampling_ratio/min": 0.7861571907997131,
|
|
"sampling/sampling_logp_difference/max": 0.276343834400177,
|
|
"sampling/sampling_logp_difference/mean": 0.021251189522445202,
|
|
"step": 7140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.5,
|
|
"completions/max_terminated_length": 255.5,
|
|
"completions/mean_length": 186.8125,
|
|
"completions/mean_terminated_length": 186.8125,
|
|
"completions/min_length": 132.3,
|
|
"completions/min_terminated_length": 132.3,
|
|
"entropy": 1.1050178408622742,
|
|
"epoch": 0.6573503723453158,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1518331915140152,
|
|
"kl": 0.03278354926151224,
|
|
"learning_rate": 4.619305319191975e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 31063193.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.40437457263469695,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.40437457859516146,
|
|
"sampling/importance_sampling_ratio/max": 1.3489197373390198,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003912448883057,
|
|
"sampling/importance_sampling_ratio/min": 0.7802765369415283,
|
|
"sampling/sampling_logp_difference/max": 0.3191341161727905,
|
|
"sampling/sampling_logp_difference/mean": 0.022332103736698628,
|
|
"step": 7150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.8,
|
|
"completions/max_terminated_length": 291.8,
|
|
"completions/mean_length": 196.4,
|
|
"completions/mean_terminated_length": 196.4,
|
|
"completions/min_length": 114.2,
|
|
"completions/min_terminated_length": 114.2,
|
|
"entropy": 1.0893623948097229,
|
|
"epoch": 0.6582697434954491,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.1559673249721527,
|
|
"kl": 0.05480624116025865,
|
|
"learning_rate": 4.617960294022717e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 31107273.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.1974882572889328,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.19748826324939728,
|
|
"sampling/importance_sampling_ratio/max": 1.2473576664924622,
|
|
"sampling/importance_sampling_ratio/mean": 1.000143676996231,
|
|
"sampling/importance_sampling_ratio/min": 0.7711097955703735,
|
|
"sampling/sampling_logp_difference/max": 0.2689287900924683,
|
|
"sampling/sampling_logp_difference/mean": 0.021457194723188876,
|
|
"step": 7160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.9,
|
|
"completions/max_terminated_length": 279.9,
|
|
"completions/mean_length": 204.0875,
|
|
"completions/mean_terminated_length": 204.0875,
|
|
"completions/min_length": 143.6,
|
|
"completions/min_terminated_length": 143.6,
|
|
"entropy": 1.0858058348298072,
|
|
"epoch": 0.6591891146455824,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12401717156171799,
|
|
"kl": 0.04866970150033012,
|
|
"learning_rate": 4.616613093538461e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 31152624.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.3483739882707596,
|
|
"rewards/reward_fn/mean": 0.375,
|
|
"rewards/reward_fn/std": 0.34837399423122406,
|
|
"sampling/importance_sampling_ratio/max": 1.2619606018066407,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003080785274505,
|
|
"sampling/importance_sampling_ratio/min": 0.7845691561698913,
|
|
"sampling/sampling_logp_difference/max": 0.2728304147720337,
|
|
"sampling/sampling_logp_difference/mean": 0.02141085732728243,
|
|
"step": 7170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 264.8,
|
|
"completions/max_terminated_length": 264.8,
|
|
"completions/mean_length": 198.625,
|
|
"completions/mean_terminated_length": 198.625,
|
|
"completions/min_length": 126.7,
|
|
"completions/min_terminated_length": 126.7,
|
|
"entropy": 1.0963338486850263,
|
|
"epoch": 0.6601084857957157,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.13831183314323425,
|
|
"kl": 0.06449027534108609,
|
|
"learning_rate": 4.615263719122891e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 31194042.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.3409078598022461,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.3409078598022461,
|
|
"sampling/importance_sampling_ratio/max": 1.2703498601913452,
|
|
"sampling/importance_sampling_ratio/mean": 1.000065553188324,
|
|
"sampling/importance_sampling_ratio/min": 0.7917282342910766,
|
|
"sampling/sampling_logp_difference/max": 0.267667293548584,
|
|
"sampling/sampling_logp_difference/mean": 0.021503900922834874,
|
|
"step": 7180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.3,
|
|
"completions/max_terminated_length": 282.3,
|
|
"completions/mean_length": 190.1,
|
|
"completions/mean_terminated_length": 190.1,
|
|
"completions/min_length": 121.8,
|
|
"completions/min_terminated_length": 121.8,
|
|
"entropy": 1.0183037899434566,
|
|
"epoch": 0.6610278569458491,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.13874197006225586,
|
|
"kl": 0.04413658081321046,
|
|
"learning_rate": 4.613912172161923e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 31235002.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.1414213538169861,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.1414213538169861,
|
|
"sampling/importance_sampling_ratio/max": 1.2225879907608033,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004299223423003,
|
|
"sampling/importance_sampling_ratio/min": 0.7879893839359283,
|
|
"sampling/sampling_logp_difference/max": 0.2429264783859253,
|
|
"sampling/sampling_logp_difference/mean": 0.021308378875255586,
|
|
"step": 7190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.9,
|
|
"completions/max_terminated_length": 274.9,
|
|
"completions/mean_length": 193.475,
|
|
"completions/mean_terminated_length": 193.475,
|
|
"completions/min_length": 128.4,
|
|
"completions/min_terminated_length": 128.4,
|
|
"entropy": 0.9930923976004123,
|
|
"epoch": 0.6619472280959824,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.2710089683532715,
|
|
"kl": 0.04803750627906993,
|
|
"learning_rate": 4.612558454043702e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 31277808.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.3820319950580597,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.38203200697898865,
|
|
"sampling/importance_sampling_ratio/max": 1.261991024017334,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001124501228333,
|
|
"sampling/importance_sampling_ratio/min": 0.783654797077179,
|
|
"sampling/sampling_logp_difference/max": 0.2786588311195374,
|
|
"sampling/sampling_logp_difference/mean": 0.020226932782679796,
|
|
"step": 7200
|
|
},
|
|
{
|
|
"epoch": 0.6619472280959824,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 284.48342059336824,
|
|
"eval_completions/max_terminated_length": 283.14659685863876,
|
|
"eval_completions/mean_length": 198.51221640488657,
|
|
"eval_completions/mean_terminated_length": 198.1925643734491,
|
|
"eval_completions/min_length": 127.1239092495637,
|
|
"eval_completions/min_terminated_length": 127.1239092495637,
|
|
"eval_entropy": 1.064619136990469,
|
|
"eval_frac_reward_zero_std": 0.31239092495637,
|
|
"eval_kl": 0.0498309459010656,
|
|
"eval_loss": 0.0026732217520475388,
|
|
"eval_num_tokens": 31277808.0,
|
|
"eval_reward": 0.06489965095986039,
|
|
"eval_reward_std": 0.3346925450141101,
|
|
"eval_rewards/reward_fn/mean": 0.06489965095986039,
|
|
"eval_rewards/reward_fn/std": 0.3346925534398976,
|
|
"eval_runtime": 1667.7373,
|
|
"eval_samples_per_second": 0.344,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2635605285305003,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000404038354365,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7779044504773138,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27392785368165423,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02119820735971549,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 7200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.6,
|
|
"completions/max_terminated_length": 303.6,
|
|
"completions/mean_length": 211.025,
|
|
"completions/mean_terminated_length": 211.025,
|
|
"completions/min_length": 139.1,
|
|
"completions/min_terminated_length": 139.1,
|
|
"entropy": 1.0055617660284042,
|
|
"epoch": 0.6628665992461157,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.493412017822266,
|
|
"kl": 0.05891411122283898,
|
|
"learning_rate": 4.611202566158607e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 31321546.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.29661122560501096,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.29661123752593993,
|
|
"sampling/importance_sampling_ratio/max": 1.28914954662323,
|
|
"sampling/importance_sampling_ratio/mean": 1.000070708990097,
|
|
"sampling/importance_sampling_ratio/min": 0.7822191119194031,
|
|
"sampling/sampling_logp_difference/max": 0.29209272265434266,
|
|
"sampling/sampling_logp_difference/mean": 0.020206468179821968,
|
|
"step": 7210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.2,
|
|
"completions/max_terminated_length": 301.2,
|
|
"completions/mean_length": 209.875,
|
|
"completions/mean_terminated_length": 209.875,
|
|
"completions/min_length": 122.9,
|
|
"completions/min_terminated_length": 122.9,
|
|
"entropy": 1.0289199404418468,
|
|
"epoch": 0.663785970396249,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.5265278816223145,
|
|
"kl": 0.04613122205482796,
|
|
"learning_rate": 4.609844509899243e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 31366680.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.3265853762626648,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.3265853762626648,
|
|
"sampling/importance_sampling_ratio/max": 1.2539484739303588,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000987112522126,
|
|
"sampling/importance_sampling_ratio/min": 0.7771053314208984,
|
|
"sampling/sampling_logp_difference/max": 0.26145464181900024,
|
|
"sampling/sampling_logp_difference/mean": 0.02051695678383112,
|
|
"step": 7220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 277.1,
|
|
"completions/max_terminated_length": 267.1,
|
|
"completions/mean_length": 197.225,
|
|
"completions/mean_terminated_length": 194.93214416503906,
|
|
"completions/min_length": 132.2,
|
|
"completions/min_terminated_length": 132.2,
|
|
"entropy": 1.0228578120470047,
|
|
"epoch": 0.6647053415463823,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.16433332860469818,
|
|
"kl": 0.04508859519264661,
|
|
"learning_rate": 4.6084842866604426e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 31407482.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.37037220001220705,
|
|
"rewards/reward_fn/mean": 0.21875,
|
|
"rewards/reward_fn/std": 0.3703722059726715,
|
|
"sampling/importance_sampling_ratio/max": 1.2638112306594849,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997063279151917,
|
|
"sampling/importance_sampling_ratio/min": 0.7383287847042084,
|
|
"sampling/sampling_logp_difference/max": 0.3096863508224487,
|
|
"sampling/sampling_logp_difference/mean": 0.021090848557651042,
|
|
"step": 7230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.9,
|
|
"completions/max_terminated_length": 291.9,
|
|
"completions/mean_length": 205.8,
|
|
"completions/mean_terminated_length": 205.8,
|
|
"completions/min_length": 139.5,
|
|
"completions/min_terminated_length": 139.5,
|
|
"entropy": 1.0304714642465114,
|
|
"epoch": 0.6656247126965156,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.23519223928451538,
|
|
"kl": 0.05572328233392909,
|
|
"learning_rate": 4.607121897839264e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 31453018.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.2756610929965973,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.27566109895706176,
|
|
"sampling/importance_sampling_ratio/max": 1.2423957347869874,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003386080265044,
|
|
"sampling/importance_sampling_ratio/min": 0.7729002416133881,
|
|
"sampling/sampling_logp_difference/max": 0.2709189295768738,
|
|
"sampling/sampling_logp_difference/mean": 0.02015252448618412,
|
|
"step": 7240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.1,
|
|
"completions/max_terminated_length": 275.1,
|
|
"completions/mean_length": 204.5625,
|
|
"completions/mean_terminated_length": 204.5625,
|
|
"completions/min_length": 134.4,
|
|
"completions/min_terminated_length": 134.4,
|
|
"entropy": 1.0800440780818463,
|
|
"epoch": 0.6665440838466489,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.17569485306739807,
|
|
"kl": 0.06364223085111007,
|
|
"learning_rate": 4.605757344834989e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 31496543.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.20411193668842315,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.20411194264888763,
|
|
"sampling/importance_sampling_ratio/max": 1.2897516369819642,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006681382656097,
|
|
"sampling/importance_sampling_ratio/min": 0.7404118180274963,
|
|
"sampling/sampling_logp_difference/max": 0.3246050179004669,
|
|
"sampling/sampling_logp_difference/mean": 0.02191933020949364,
|
|
"step": 7250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.9,
|
|
"completions/max_terminated_length": 271.9,
|
|
"completions/mean_length": 186.1875,
|
|
"completions/mean_terminated_length": 186.1875,
|
|
"completions/min_length": 120.4,
|
|
"completions/min_terminated_length": 120.4,
|
|
"entropy": 1.0374748148024082,
|
|
"epoch": 0.6674634549967822,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.727726936340332,
|
|
"kl": 0.04095509689068422,
|
|
"learning_rate": 4.604390629049125e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 31540822.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.3282748609781265,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.328274866938591,
|
|
"sampling/importance_sampling_ratio/max": 1.3150431394577027,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001801908016206,
|
|
"sampling/importance_sampling_ratio/min": 0.7840251088142395,
|
|
"sampling/sampling_logp_difference/max": 0.30605231523513793,
|
|
"sampling/sampling_logp_difference/mean": 0.021714341826736927,
|
|
"step": 7260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.5,
|
|
"completions/max_terminated_length": 311.5,
|
|
"completions/mean_length": 213.075,
|
|
"completions/mean_terminated_length": 213.075,
|
|
"completions/min_length": 130.8,
|
|
"completions/min_terminated_length": 130.8,
|
|
"entropy": 1.042024876177311,
|
|
"epoch": 0.6683828261469155,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.499321937561035,
|
|
"kl": 0.04465296705020592,
|
|
"learning_rate": 4.603021751885397e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 31583180.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.44357921183109283,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.4435792237520218,
|
|
"sampling/importance_sampling_ratio/max": 1.2784564971923829,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999276459217071,
|
|
"sampling/importance_sampling_ratio/min": 0.7687676250934601,
|
|
"sampling/sampling_logp_difference/max": 0.2863020122051239,
|
|
"sampling/sampling_logp_difference/mean": 0.021744346618652342,
|
|
"step": 7270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.1,
|
|
"completions/max_terminated_length": 280.1,
|
|
"completions/mean_length": 195.0,
|
|
"completions/mean_terminated_length": 195.0,
|
|
"completions/min_length": 130.5,
|
|
"completions/min_terminated_length": 130.5,
|
|
"entropy": 0.9949001189321279,
|
|
"epoch": 0.6693021972970489,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.6927490234375,
|
|
"kl": 0.055850255314726385,
|
|
"learning_rate": 4.601650714749752e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 31622796.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.42896814048290255,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.42896815240383146,
|
|
"sampling/importance_sampling_ratio/max": 1.36483793258667,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003822922706604,
|
|
"sampling/importance_sampling_ratio/min": 0.7705457389354706,
|
|
"sampling/sampling_logp_difference/max": 0.327013099193573,
|
|
"sampling/sampling_logp_difference/mean": 0.02087930031120777,
|
|
"step": 7280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.6,
|
|
"completions/max_terminated_length": 255.6,
|
|
"completions/mean_length": 189.6375,
|
|
"completions/mean_terminated_length": 189.6375,
|
|
"completions/min_length": 123.9,
|
|
"completions/min_terminated_length": 123.9,
|
|
"entropy": 1.0203311927616596,
|
|
"epoch": 0.6702215684471822,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1278441995382309,
|
|
"kl": 0.04536830126889981,
|
|
"learning_rate": 4.600277519050355e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 31662711.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.27773033976554873,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.2777303636074066,
|
|
"sampling/importance_sampling_ratio/max": 1.226729953289032,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999350786209107,
|
|
"sampling/importance_sampling_ratio/min": 0.8006794214248657,
|
|
"sampling/sampling_logp_difference/max": 0.23733015060424806,
|
|
"sampling/sampling_logp_difference/mean": 0.020400801673531532,
|
|
"step": 7290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.0,
|
|
"completions/max_terminated_length": 304.0,
|
|
"completions/mean_length": 203.6375,
|
|
"completions/mean_terminated_length": 203.6375,
|
|
"completions/min_length": 122.6,
|
|
"completions/min_terminated_length": 122.6,
|
|
"entropy": 1.1139685682952405,
|
|
"epoch": 0.6711409395973155,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13240955770015717,
|
|
"kl": 0.051546607888303696,
|
|
"learning_rate": 4.598902166197589e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 31706122.0,
|
|
"reward": -0.14375,
|
|
"reward_std": 0.2735445201396942,
|
|
"rewards/reward_fn/mean": -0.14375,
|
|
"rewards/reward_fn/std": 0.2735445320606232,
|
|
"sampling/importance_sampling_ratio/max": 1.3180628299713135,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000078141689301,
|
|
"sampling/importance_sampling_ratio/min": 0.7910449624061584,
|
|
"sampling/sampling_logp_difference/max": 0.2981088161468506,
|
|
"sampling/sampling_logp_difference/mean": 0.021368534304201603,
|
|
"step": 7300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 318.2,
|
|
"completions/max_terminated_length": 296.1,
|
|
"completions/mean_length": 210.575,
|
|
"completions/mean_terminated_length": 206.87857208251953,
|
|
"completions/min_length": 131.4,
|
|
"completions/min_terminated_length": 131.4,
|
|
"entropy": 1.1729208558797837,
|
|
"epoch": 0.6720603107474488,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 3.9134163856506348,
|
|
"kl": 0.04998117060167715,
|
|
"learning_rate": 4.597524657604049e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 31753496.0,
|
|
"reward": -0.01875,
|
|
"reward_std": 0.1864342659711838,
|
|
"rewards/reward_fn/mean": -0.01875,
|
|
"rewards/reward_fn/std": 0.18643427193164824,
|
|
"sampling/importance_sampling_ratio/max": 1.2764506816864014,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004078209400178,
|
|
"sampling/importance_sampling_ratio/min": 0.7668221116065979,
|
|
"sampling/sampling_logp_difference/max": 0.2980937361717224,
|
|
"sampling/sampling_logp_difference/mean": 0.02186319902539253,
|
|
"step": 7310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.9,
|
|
"completions/max_terminated_length": 297.9,
|
|
"completions/mean_length": 215.1625,
|
|
"completions/mean_terminated_length": 215.1625,
|
|
"completions/min_length": 153.2,
|
|
"completions/min_terminated_length": 153.2,
|
|
"entropy": 1.0700261563062667,
|
|
"epoch": 0.6729796818975821,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11135904490947723,
|
|
"kl": 0.05424491100711748,
|
|
"learning_rate": 4.5961449946845495e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 31799517.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.28516874015331267,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.28516875207424164,
|
|
"sampling/importance_sampling_ratio/max": 1.2372430443763733,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000016927719115,
|
|
"sampling/importance_sampling_ratio/min": 0.790747195482254,
|
|
"sampling/sampling_logp_difference/max": 0.24547867774963378,
|
|
"sampling/sampling_logp_difference/mean": 0.020578095875680446,
|
|
"step": 7320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 318.1,
|
|
"completions/max_terminated_length": 318.1,
|
|
"completions/mean_length": 220.6625,
|
|
"completions/mean_terminated_length": 220.6625,
|
|
"completions/min_length": 134.7,
|
|
"completions/min_terminated_length": 134.7,
|
|
"entropy": 1.1330191358923911,
|
|
"epoch": 0.6738990530477154,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12394562363624573,
|
|
"kl": 0.04857685996685177,
|
|
"learning_rate": 4.594763178856113e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 31844370.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.2504029482603073,
|
|
"sampling/importance_sampling_ratio/max": 1.2333351254463196,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001760303974152,
|
|
"sampling/importance_sampling_ratio/min": 0.7773239195346833,
|
|
"sampling/sampling_logp_difference/max": 0.2579160749912262,
|
|
"sampling/sampling_logp_difference/mean": 0.022077248059213163,
|
|
"step": 7330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 320.6,
|
|
"completions/max_terminated_length": 320.6,
|
|
"completions/mean_length": 232.1125,
|
|
"completions/mean_terminated_length": 232.1125,
|
|
"completions/min_length": 159.4,
|
|
"completions/min_terminated_length": 159.4,
|
|
"entropy": 1.0977382034063339,
|
|
"epoch": 0.6748184241978487,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 6.417923450469971,
|
|
"kl": 0.05118432566523552,
|
|
"learning_rate": 4.593379211537977e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 31889987.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.2292030245065689,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.22920303642749787,
|
|
"sampling/importance_sampling_ratio/max": 1.2419498562812805,
|
|
"sampling/importance_sampling_ratio/mean": 1.000518250465393,
|
|
"sampling/importance_sampling_ratio/min": 0.7248602390289307,
|
|
"sampling/sampling_logp_difference/max": 0.3369430899620056,
|
|
"sampling/sampling_logp_difference/mean": 0.02117815352976322,
|
|
"step": 7340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 317.2,
|
|
"completions/max_terminated_length": 317.2,
|
|
"completions/mean_length": 222.5,
|
|
"completions/mean_terminated_length": 222.5,
|
|
"completions/min_length": 144.3,
|
|
"completions/min_terminated_length": 144.3,
|
|
"entropy": 1.1368062272667885,
|
|
"epoch": 0.675737795347982,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.15781785547733307,
|
|
"kl": 0.034471037273760885,
|
|
"learning_rate": 4.591993094151586e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 31937627.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.1595182627439499,
|
|
"rewards/reward_fn/mean": 0.3375,
|
|
"rewards/reward_fn/std": 0.1595182627439499,
|
|
"sampling/importance_sampling_ratio/max": 1.267795467376709,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997016847133636,
|
|
"sampling/importance_sampling_ratio/min": 0.7713879704475403,
|
|
"sampling/sampling_logp_difference/max": 0.2952385932207108,
|
|
"sampling/sampling_logp_difference/mean": 0.02182802725583315,
|
|
"step": 7350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.5,
|
|
"completions/max_terminated_length": 294.5,
|
|
"completions/mean_length": 215.725,
|
|
"completions/mean_terminated_length": 215.725,
|
|
"completions/min_length": 146.5,
|
|
"completions/min_terminated_length": 146.5,
|
|
"entropy": 1.0568891540169716,
|
|
"epoch": 0.6766571664981152,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.203394412994385,
|
|
"kl": 0.05546131924493238,
|
|
"learning_rate": 4.590604828120593e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 31981757.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.3423954129219055,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.3423954248428345,
|
|
"sampling/importance_sampling_ratio/max": 1.2456037282943726,
|
|
"sampling/importance_sampling_ratio/mean": 1.000145787000656,
|
|
"sampling/importance_sampling_ratio/min": 0.795643550157547,
|
|
"sampling/sampling_logp_difference/max": 0.23592472672462464,
|
|
"sampling/sampling_logp_difference/mean": 0.020911729894578457,
|
|
"step": 7360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.5,
|
|
"completions/max_terminated_length": 273.5,
|
|
"completions/mean_length": 211.875,
|
|
"completions/mean_terminated_length": 211.875,
|
|
"completions/min_length": 151.1,
|
|
"completions/min_terminated_length": 151.1,
|
|
"entropy": 0.9900574635714292,
|
|
"epoch": 0.6775765376482485,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.566556453704834,
|
|
"kl": 0.049262395867845046,
|
|
"learning_rate": 4.589214414870858e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 32029371.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.300939616560936,
|
|
"rewards/reward_fn/mean": 0.2125,
|
|
"rewards/reward_fn/std": 0.300939616560936,
|
|
"sampling/importance_sampling_ratio/max": 1.2418309807777406,
|
|
"sampling/importance_sampling_ratio/mean": 1.000444734096527,
|
|
"sampling/importance_sampling_ratio/min": 0.7783786952495575,
|
|
"sampling/sampling_logp_difference/max": 0.2679599463939667,
|
|
"sampling/sampling_logp_difference/mean": 0.020084110647439958,
|
|
"step": 7370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.9,
|
|
"completions/max_terminated_length": 299.9,
|
|
"completions/mean_length": 205.7875,
|
|
"completions/mean_terminated_length": 205.7875,
|
|
"completions/min_length": 128.4,
|
|
"completions/min_terminated_length": 128.4,
|
|
"entropy": 0.9073899272829294,
|
|
"epoch": 0.678495908798382,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 3.5518264770507812,
|
|
"kl": 0.036938689762610014,
|
|
"learning_rate": 4.58782185583045e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 32072874.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4173794865608215,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.41737951040267945,
|
|
"sampling/importance_sampling_ratio/max": 1.2959476947784423,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002209484577178,
|
|
"sampling/importance_sampling_ratio/min": 0.7928396224975586,
|
|
"sampling/sampling_logp_difference/max": 0.2790764331817627,
|
|
"sampling/sampling_logp_difference/mean": 0.01888665952719748,
|
|
"step": 7380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.4,
|
|
"completions/max_terminated_length": 283.4,
|
|
"completions/mean_length": 192.5625,
|
|
"completions/mean_terminated_length": 192.5625,
|
|
"completions/min_length": 128.6,
|
|
"completions/min_terminated_length": 128.6,
|
|
"entropy": 1.0405104793608189,
|
|
"epoch": 0.6794152799485152,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.13286620378494263,
|
|
"kl": 0.04277502251788974,
|
|
"learning_rate": 4.5864271524296355e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 32112607.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/reward_fn/mean": 0.3375,
|
|
"rewards/reward_fn/std": 0.2150476098060608,
|
|
"sampling/importance_sampling_ratio/max": 1.2894740223884582,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004708468914032,
|
|
"sampling/importance_sampling_ratio/min": 0.7762864589691162,
|
|
"sampling/sampling_logp_difference/max": 0.2877599477767944,
|
|
"sampling/sampling_logp_difference/mean": 0.020924269407987594,
|
|
"step": 7390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.1,
|
|
"completions/max_terminated_length": 312.1,
|
|
"completions/mean_length": 202.5875,
|
|
"completions/mean_terminated_length": 202.5875,
|
|
"completions/min_length": 123.8,
|
|
"completions/min_terminated_length": 123.8,
|
|
"entropy": 1.0661517322063445,
|
|
"epoch": 0.6803346510986485,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.13939781486988068,
|
|
"kl": 0.06156819289317354,
|
|
"learning_rate": 4.585030306100887e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 32153334.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.1414213538169861,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.1414213538169861,
|
|
"sampling/importance_sampling_ratio/max": 1.2577814698219298,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998873054981232,
|
|
"sampling/importance_sampling_ratio/min": 0.7967040300369262,
|
|
"sampling/sampling_logp_difference/max": 0.2475142478942871,
|
|
"sampling/sampling_logp_difference/mean": 0.021554626151919366,
|
|
"step": 7400
|
|
},
|
|
{
|
|
"epoch": 0.6803346510986485,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 290.28970331588135,
|
|
"eval_completions/max_terminated_length": 289.32111692844677,
|
|
"eval_completions/mean_length": 205.44742582897032,
|
|
"eval_completions/mean_terminated_length": 205.1254052500034,
|
|
"eval_completions/min_length": 134.56020942408378,
|
|
"eval_completions/min_terminated_length": 134.56020942408378,
|
|
"eval_entropy": 1.0888234543342656,
|
|
"eval_frac_reward_zero_std": 0.343804537521815,
|
|
"eval_kl": 0.05033627688806719,
|
|
"eval_loss": 0.0026131882332265377,
|
|
"eval_num_tokens": 32153334.0,
|
|
"eval_reward": 0.061845549738219895,
|
|
"eval_reward_std": 0.31748389781665637,
|
|
"eval_rewards/reward_fn/mean": 0.061845549738219895,
|
|
"eval_rewards/reward_fn/std": 0.31748390780277486,
|
|
"eval_runtime": 1698.8002,
|
|
"eval_samples_per_second": 0.337,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2657911800381192,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000523089530373,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7797359454902263,
|
|
"eval_sampling/sampling_logp_difference/max": 0.273970496176009,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02093242133852179,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 7400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.1,
|
|
"completions/max_terminated_length": 268.1,
|
|
"completions/mean_length": 201.4,
|
|
"completions/mean_terminated_length": 201.4,
|
|
"completions/min_length": 135.6,
|
|
"completions/min_terminated_length": 135.6,
|
|
"entropy": 1.1320892922580241,
|
|
"epoch": 0.6812540222487818,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.608371734619141,
|
|
"kl": 0.05768209486268461,
|
|
"learning_rate": 4.5836313182788804e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 32194814.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.4007413417100906,
|
|
"rewards/reward_fn/mean": 0.2875,
|
|
"rewards/reward_fn/std": 0.40074135959148405,
|
|
"sampling/importance_sampling_ratio/max": 1.2715680241584777,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999514818191528,
|
|
"sampling/importance_sampling_ratio/min": 0.7736750602722168,
|
|
"sampling/sampling_logp_difference/max": 0.27415614724159243,
|
|
"sampling/sampling_logp_difference/mean": 0.022923282533884048,
|
|
"step": 7410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 329.9,
|
|
"completions/max_terminated_length": 303.9,
|
|
"completions/mean_length": 210.825,
|
|
"completions/mean_terminated_length": 206.81964416503905,
|
|
"completions/min_length": 128.2,
|
|
"completions/min_terminated_length": 128.2,
|
|
"entropy": 1.0314208410680294,
|
|
"epoch": 0.6821733933989151,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.2035738080739975,
|
|
"kl": 0.06656519437674432,
|
|
"learning_rate": 4.582230190400486e-06,
|
|
"loss": 0.0035,
|
|
"num_tokens": 32241760.0,
|
|
"reward": -0.03125,
|
|
"reward_std": 0.26260885298252107,
|
|
"rewards/reward_fn/mean": -0.03125,
|
|
"rewards/reward_fn/std": 0.26260886490345003,
|
|
"sampling/importance_sampling_ratio/max": 1.26887948513031,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000440657138825,
|
|
"sampling/importance_sampling_ratio/min": 0.7858240187168122,
|
|
"sampling/sampling_logp_difference/max": 0.25771403312683105,
|
|
"sampling/sampling_logp_difference/mean": 0.020971113443374635,
|
|
"step": 7420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.3,
|
|
"completions/max_terminated_length": 289.3,
|
|
"completions/mean_length": 193.475,
|
|
"completions/mean_terminated_length": 193.475,
|
|
"completions/min_length": 115.1,
|
|
"completions/min_terminated_length": 115.1,
|
|
"entropy": 0.9514463262632489,
|
|
"epoch": 0.6830927645490484,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.9331817626953125,
|
|
"kl": 0.06294347807997838,
|
|
"learning_rate": 4.580826923904774e-06,
|
|
"loss": 0.0034,
|
|
"num_tokens": 32285414.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.35585459768772126,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.35585459768772126,
|
|
"sampling/importance_sampling_ratio/max": 1.2565185666084289,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996947526931763,
|
|
"sampling/importance_sampling_ratio/min": 0.7628743946552277,
|
|
"sampling/sampling_logp_difference/max": 0.28076925277709963,
|
|
"sampling/sampling_logp_difference/mean": 0.01992936562746763,
|
|
"step": 7430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 283.9,
|
|
"completions/max_terminated_length": 266.1,
|
|
"completions/mean_length": 185.7375,
|
|
"completions/mean_terminated_length": 181.9607147216797,
|
|
"completions/min_length": 92.1,
|
|
"completions/min_terminated_length": 92.1,
|
|
"entropy": 1.0744081519544124,
|
|
"epoch": 0.6840121356991817,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 8.312846183776855,
|
|
"kl": 0.03791194562218152,
|
|
"learning_rate": 4.579421520233013e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 32328881.0,
|
|
"reward": 0.04375,
|
|
"reward_std": 0.21255126893520354,
|
|
"rewards/reward_fn/mean": 0.04375,
|
|
"rewards/reward_fn/std": 0.21255126893520354,
|
|
"sampling/importance_sampling_ratio/max": 1.2457831263542176,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004271984100341,
|
|
"sampling/importance_sampling_ratio/min": 0.80016610622406,
|
|
"sampling/sampling_logp_difference/max": 0.23352012634277344,
|
|
"sampling/sampling_logp_difference/mean": 0.0215287521481514,
|
|
"step": 7440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.0,
|
|
"completions/max_terminated_length": 297.0,
|
|
"completions/mean_length": 202.575,
|
|
"completions/mean_terminated_length": 202.575,
|
|
"completions/min_length": 139.1,
|
|
"completions/min_terminated_length": 139.1,
|
|
"entropy": 1.132238607108593,
|
|
"epoch": 0.684931506849315,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.909250736236572,
|
|
"kl": 0.04190785058890469,
|
|
"learning_rate": 4.578013980828664e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 32376975.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.48442131876945493,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.48442131876945493,
|
|
"sampling/importance_sampling_ratio/max": 1.251918911933899,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994283258914948,
|
|
"sampling/importance_sampling_ratio/min": 0.7680756390094757,
|
|
"sampling/sampling_logp_difference/max": 0.295600438117981,
|
|
"sampling/sampling_logp_difference/mean": 0.02196041587740183,
|
|
"step": 7450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 338.1,
|
|
"completions/max_terminated_length": 338.1,
|
|
"completions/mean_length": 228.4,
|
|
"completions/mean_terminated_length": 228.4,
|
|
"completions/min_length": 126.1,
|
|
"completions/min_terminated_length": 126.1,
|
|
"entropy": 1.2107975255697965,
|
|
"epoch": 0.6858508779994483,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.11257170140743256,
|
|
"kl": 0.05027083324966952,
|
|
"learning_rate": 4.5766043071373845e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 32421679.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.3745658665895462,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.37456587255001067,
|
|
"sampling/importance_sampling_ratio/max": 1.3062814235687257,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996409952640534,
|
|
"sampling/importance_sampling_ratio/min": 0.7797921299934387,
|
|
"sampling/sampling_logp_difference/max": 0.2978152811527252,
|
|
"sampling/sampling_logp_difference/mean": 0.023141766153275966,
|
|
"step": 7460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 322.8,
|
|
"completions/max_terminated_length": 319.2,
|
|
"completions/mean_length": 205.775,
|
|
"completions/mean_terminated_length": 202.9357147216797,
|
|
"completions/min_length": 119.4,
|
|
"completions/min_terminated_length": 119.4,
|
|
"entropy": 1.064218432456255,
|
|
"epoch": 0.6867702491495817,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.38142204284668,
|
|
"kl": 0.04115309783956036,
|
|
"learning_rate": 4.57519250060702e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 32464917.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.34506991803646087,
|
|
"rewards/reward_fn/mean": 0.03125,
|
|
"rewards/reward_fn/std": 0.34506991803646087,
|
|
"sampling/importance_sampling_ratio/max": 1.2449893236160279,
|
|
"sampling/importance_sampling_ratio/mean": 1.000265634059906,
|
|
"sampling/importance_sampling_ratio/min": 0.7682155489921569,
|
|
"sampling/sampling_logp_difference/max": 0.2762560248374939,
|
|
"sampling/sampling_logp_difference/mean": 0.021231625229120255,
|
|
"step": 7470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.1,
|
|
"completions/max_terminated_length": 295.1,
|
|
"completions/mean_length": 198.3125,
|
|
"completions/mean_terminated_length": 198.3125,
|
|
"completions/min_length": 119.3,
|
|
"completions/min_terminated_length": 119.3,
|
|
"entropy": 1.0488322123885154,
|
|
"epoch": 0.687689620299715,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.424997806549072,
|
|
"kl": 0.03883749302476645,
|
|
"learning_rate": 4.573778562687612e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 32506270.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.44697387516498566,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.44697387516498566,
|
|
"sampling/importance_sampling_ratio/max": 1.2687963485717773,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997938632965088,
|
|
"sampling/importance_sampling_ratio/min": 0.7802180767059326,
|
|
"sampling/sampling_logp_difference/max": 0.26483807563781736,
|
|
"sampling/sampling_logp_difference/mean": 0.021511211432516573,
|
|
"step": 7480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.1,
|
|
"completions/max_terminated_length": 301.1,
|
|
"completions/mean_length": 219.975,
|
|
"completions/mean_terminated_length": 219.975,
|
|
"completions/min_length": 132.8,
|
|
"completions/min_terminated_length": 132.8,
|
|
"entropy": 1.1070709958672524,
|
|
"epoch": 0.6886089914498483,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.352287769317627,
|
|
"kl": 0.04235333006363362,
|
|
"learning_rate": 4.572362494831386e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 32549756.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.3093190997838974,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.30931911170482634,
|
|
"sampling/importance_sampling_ratio/max": 1.2543426632881165,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003794074058532,
|
|
"sampling/importance_sampling_ratio/min": 0.7820522606372833,
|
|
"sampling/sampling_logp_difference/max": 0.26365768909454346,
|
|
"sampling/sampling_logp_difference/mean": 0.021890545822679998,
|
|
"step": 7490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.1,
|
|
"completions/max_terminated_length": 273.1,
|
|
"completions/mean_length": 202.05,
|
|
"completions/mean_terminated_length": 202.05,
|
|
"completions/min_length": 123.3,
|
|
"completions/min_terminated_length": 123.3,
|
|
"entropy": 1.1775474093854428,
|
|
"epoch": 0.6895283625999816,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.267361640930176,
|
|
"kl": 0.058694390836171804,
|
|
"learning_rate": 4.5709442984927564e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 32592440.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.4013115704059601,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4013115882873535,
|
|
"sampling/importance_sampling_ratio/max": 1.2642083883285522,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999862372875213,
|
|
"sampling/importance_sampling_ratio/min": 0.7694486200809478,
|
|
"sampling/sampling_logp_difference/max": 0.2832889914512634,
|
|
"sampling/sampling_logp_difference/mean": 0.022359587252140045,
|
|
"step": 7500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 341.2,
|
|
"completions/max_terminated_length": 341.2,
|
|
"completions/mean_length": 227.1625,
|
|
"completions/mean_terminated_length": 227.1625,
|
|
"completions/min_length": 162.7,
|
|
"completions/min_terminated_length": 162.7,
|
|
"entropy": 1.1424855321645737,
|
|
"epoch": 0.6904477337501149,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14225175976753235,
|
|
"kl": 0.051977888692636046,
|
|
"learning_rate": 4.5695239751283275e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 32640413.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.3202547788619995,
|
|
"sampling/importance_sampling_ratio/max": 1.2743274807929992,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995707333087921,
|
|
"sampling/importance_sampling_ratio/min": 0.7844434559345246,
|
|
"sampling/sampling_logp_difference/max": 0.2723142385482788,
|
|
"sampling/sampling_logp_difference/mean": 0.021843813173472882,
|
|
"step": 7510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.8,
|
|
"completions/max_terminated_length": 306.8,
|
|
"completions/mean_length": 217.2875,
|
|
"completions/mean_terminated_length": 217.2875,
|
|
"completions/min_length": 148.4,
|
|
"completions/min_terminated_length": 148.4,
|
|
"entropy": 1.1221212826669216,
|
|
"epoch": 0.6913671049002482,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.902036190032959,
|
|
"kl": 0.04671697786543518,
|
|
"learning_rate": 4.568101526196882e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 32684084.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.4355591177940369,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.4355591356754303,
|
|
"sampling/importance_sampling_ratio/max": 1.2734981060028077,
|
|
"sampling/importance_sampling_ratio/mean": 0.999754786491394,
|
|
"sampling/importance_sampling_ratio/min": 0.7822802126407623,
|
|
"sampling/sampling_logp_difference/max": 0.27042909860610964,
|
|
"sampling/sampling_logp_difference/mean": 0.021833948232233523,
|
|
"step": 7520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.1,
|
|
"completions/max_terminated_length": 295.1,
|
|
"completions/mean_length": 211.9,
|
|
"completions/mean_terminated_length": 211.9,
|
|
"completions/min_length": 149.4,
|
|
"completions/min_terminated_length": 149.4,
|
|
"entropy": 1.0731165552511812,
|
|
"epoch": 0.6922864760503815,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.176202297210693,
|
|
"kl": 0.038268228416563944,
|
|
"learning_rate": 4.566676953159392e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 32729092.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.3311904340982437,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.3311904460191727,
|
|
"sampling/importance_sampling_ratio/max": 1.265696358680725,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003490507602693,
|
|
"sampling/importance_sampling_ratio/min": 0.7816310703754425,
|
|
"sampling/sampling_logp_difference/max": 0.26671168208122253,
|
|
"sampling/sampling_logp_difference/mean": 0.020541245769709348,
|
|
"step": 7530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.4,
|
|
"completions/max_terminated_length": 303.4,
|
|
"completions/mean_length": 225.2375,
|
|
"completions/mean_terminated_length": 225.2375,
|
|
"completions/min_length": 173.1,
|
|
"completions/min_terminated_length": 173.1,
|
|
"entropy": 1.1765621855854989,
|
|
"epoch": 0.6932058472005148,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.13530386984348297,
|
|
"kl": 0.039450769958784805,
|
|
"learning_rate": 4.565250257479004e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 32774247.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.40760951340198515,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.40760951936244966,
|
|
"sampling/importance_sampling_ratio/max": 1.2988221883773803,
|
|
"sampling/importance_sampling_ratio/mean": 1.000485473871231,
|
|
"sampling/importance_sampling_ratio/min": 0.7841191112995147,
|
|
"sampling/sampling_logp_difference/max": 0.2745662212371826,
|
|
"sampling/sampling_logp_difference/mean": 0.02200355865061283,
|
|
"step": 7540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.7,
|
|
"completions/max_terminated_length": 298.7,
|
|
"completions/mean_length": 222.45,
|
|
"completions/mean_terminated_length": 222.45,
|
|
"completions/min_length": 164.0,
|
|
"completions/min_terminated_length": 164.0,
|
|
"entropy": 1.1780326135456562,
|
|
"epoch": 0.6941252183506481,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1095915138721466,
|
|
"kl": 0.05131513993255794,
|
|
"learning_rate": 4.563821440621055e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 32819875.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.30439275205135347,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.3043927580118179,
|
|
"sampling/importance_sampling_ratio/max": 1.2823341965675354,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999317348003387,
|
|
"sampling/importance_sampling_ratio/min": 0.7296609789133072,
|
|
"sampling/sampling_logp_difference/max": 0.3403995394706726,
|
|
"sampling/sampling_logp_difference/mean": 0.022616959176957607,
|
|
"step": 7550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 295.7,
|
|
"completions/max_terminated_length": 268.4,
|
|
"completions/mean_length": 199.9875,
|
|
"completions/mean_terminated_length": 195.71428680419922,
|
|
"completions/min_length": 133.5,
|
|
"completions/min_terminated_length": 133.5,
|
|
"entropy": 1.0845373101532458,
|
|
"epoch": 0.6950445895007815,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.13479961454868317,
|
|
"kl": 0.03286644645268098,
|
|
"learning_rate": 4.562390504053049e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 32860586.0,
|
|
"reward": -0.05625,
|
|
"reward_std": 0.4166190758347511,
|
|
"rewards/reward_fn/mean": -0.05625,
|
|
"rewards/reward_fn/std": 0.4166190758347511,
|
|
"sampling/importance_sampling_ratio/max": 1.2547454833984375,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001095533370972,
|
|
"sampling/importance_sampling_ratio/min": 0.7857195138931274,
|
|
"sampling/sampling_logp_difference/max": 0.2574007511138916,
|
|
"sampling/sampling_logp_difference/mean": 0.02132561318576336,
|
|
"step": 7560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 319.0,
|
|
"completions/max_terminated_length": 301.9,
|
|
"completions/mean_length": 227.9875,
|
|
"completions/mean_terminated_length": 224.77678680419922,
|
|
"completions/min_length": 144.1,
|
|
"completions/min_terminated_length": 144.1,
|
|
"entropy": 1.1534547120332719,
|
|
"epoch": 0.6959639606509148,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.957430362701416,
|
|
"kl": 0.030418830597773193,
|
|
"learning_rate": 4.560957449244675e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 32906073.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.40544959008693693,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.4054496079683304,
|
|
"sampling/importance_sampling_ratio/max": 1.2866672992706298,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993079364299774,
|
|
"sampling/importance_sampling_ratio/min": 0.777455598115921,
|
|
"sampling/sampling_logp_difference/max": 0.2747867345809937,
|
|
"sampling/sampling_logp_difference/mean": 0.022157854959368707,
|
|
"step": 7570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.3,
|
|
"completions/max_terminated_length": 300.3,
|
|
"completions/mean_length": 206.8,
|
|
"completions/mean_terminated_length": 206.8,
|
|
"completions/min_length": 149.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"entropy": 1.144667898118496,
|
|
"epoch": 0.6968833318010481,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.517390727996826,
|
|
"kl": 0.03396015546750277,
|
|
"learning_rate": 4.5595222776677935e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 32946865.0,
|
|
"reward": 0.20625,
|
|
"reward_std": 0.46173239052295684,
|
|
"rewards/reward_fn/mean": 0.20625,
|
|
"rewards/reward_fn/std": 0.46173239648342135,
|
|
"sampling/importance_sampling_ratio/max": 1.2663776516914367,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000428795814513,
|
|
"sampling/importance_sampling_ratio/min": 0.777834290266037,
|
|
"sampling/sampling_logp_difference/max": 0.27124514877796174,
|
|
"sampling/sampling_logp_difference/mean": 0.022485843859612942,
|
|
"step": 7580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.3,
|
|
"completions/max_terminated_length": 297.3,
|
|
"completions/mean_length": 211.4625,
|
|
"completions/mean_terminated_length": 211.4625,
|
|
"completions/min_length": 149.2,
|
|
"completions/min_terminated_length": 149.2,
|
|
"entropy": 1.1465738616883754,
|
|
"epoch": 0.6978027029511814,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.040626525878906,
|
|
"kl": 0.04534736567875371,
|
|
"learning_rate": 4.5580849907964435e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 32987982.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.3973605424165726,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.3973605424165726,
|
|
"sampling/importance_sampling_ratio/max": 1.2816292643547058,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996797740459442,
|
|
"sampling/importance_sampling_ratio/min": 0.7559710204601288,
|
|
"sampling/sampling_logp_difference/max": 0.3027397871017456,
|
|
"sampling/sampling_logp_difference/mean": 0.022260284051299095,
|
|
"step": 7590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.1,
|
|
"completions/max_terminated_length": 271.1,
|
|
"completions/mean_length": 190.0625,
|
|
"completions/mean_terminated_length": 190.0625,
|
|
"completions/min_length": 123.8,
|
|
"completions/min_terminated_length": 123.8,
|
|
"entropy": 1.0846397157758474,
|
|
"epoch": 0.6987220741013147,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.175319194793701,
|
|
"kl": 0.05151133330073208,
|
|
"learning_rate": 4.5566455901068305e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 33029787.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.38550153374671936,
|
|
"rewards/reward_fn/mean": 0.4125,
|
|
"rewards/reward_fn/std": 0.3855015397071838,
|
|
"sampling/importance_sampling_ratio/max": 1.2769267559051514,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996841490268707,
|
|
"sampling/importance_sampling_ratio/min": 0.8086808145046234,
|
|
"sampling/sampling_logp_difference/max": 0.2563469409942627,
|
|
"sampling/sampling_logp_difference/mean": 0.02165211960673332,
|
|
"step": 7600
|
|
},
|
|
{
|
|
"epoch": 0.6987220741013147,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0006544502617801048,
|
|
"eval_completions/max_length": 282.239092495637,
|
|
"eval_completions/max_terminated_length": 281.80279232111695,
|
|
"eval_completions/mean_length": 200.5185427574171,
|
|
"eval_completions/mean_terminated_length": 200.34199705007396,
|
|
"eval_completions/min_length": 132.00523560209425,
|
|
"eval_completions/min_terminated_length": 132.00523560209425,
|
|
"eval_entropy": 1.1149459109880537,
|
|
"eval_frac_reward_zero_std": 0.29668411867364747,
|
|
"eval_kl": 0.038804631088624655,
|
|
"eval_loss": 0.0019907415844500065,
|
|
"eval_num_tokens": 33029787.0,
|
|
"eval_reward": 0.056391797556719025,
|
|
"eval_reward_std": 0.36392435577111304,
|
|
"eval_rewards/reward_fn/mean": 0.056391797556719025,
|
|
"eval_rewards/reward_fn/std": 0.36392436700549635,
|
|
"eval_runtime": 1657.7012,
|
|
"eval_samples_per_second": 0.346,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2669749332764177,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.000005006582117,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7869546036341636,
|
|
"eval_sampling/sampling_logp_difference/max": 0.26855901086517653,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020919934141061174,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 7600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 328.4,
|
|
"completions/max_terminated_length": 328.4,
|
|
"completions/mean_length": 217.9,
|
|
"completions/mean_terminated_length": 217.9,
|
|
"completions/min_length": 143.3,
|
|
"completions/min_terminated_length": 143.3,
|
|
"entropy": 1.1236586943268776,
|
|
"epoch": 0.699641445251448,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.13727843761444092,
|
|
"kl": 0.045946736662881446,
|
|
"learning_rate": 4.555204077077338e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 33073835.0,
|
|
"reward": -0.275,
|
|
"reward_std": 0.2874556124210358,
|
|
"rewards/reward_fn/mean": -0.275,
|
|
"rewards/reward_fn/std": 0.2874556124210358,
|
|
"sampling/importance_sampling_ratio/max": 1.253422498703003,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996615648269653,
|
|
"sampling/importance_sampling_ratio/min": 0.789066880941391,
|
|
"sampling/sampling_logp_difference/max": 0.2551787614822388,
|
|
"sampling/sampling_logp_difference/mean": 0.021304538100957872,
|
|
"step": 7610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.6,
|
|
"completions/max_terminated_length": 289.6,
|
|
"completions/mean_length": 205.725,
|
|
"completions/mean_terminated_length": 205.725,
|
|
"completions/min_length": 132.8,
|
|
"completions/min_terminated_length": 132.8,
|
|
"entropy": 1.130693106353283,
|
|
"epoch": 0.7005608164015813,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1048576831817627,
|
|
"kl": 0.0387039344292134,
|
|
"learning_rate": 4.553760453188513e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 33116901.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.3929676622152328,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.3929676741361618,
|
|
"sampling/importance_sampling_ratio/max": 1.256409728527069,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999854505062103,
|
|
"sampling/importance_sampling_ratio/min": 0.770319128036499,
|
|
"sampling/sampling_logp_difference/max": 0.2739866256713867,
|
|
"sampling/sampling_logp_difference/mean": 0.022002885490655898,
|
|
"step": 7620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 250.7,
|
|
"completions/max_terminated_length": 250.7,
|
|
"completions/mean_length": 182.9375,
|
|
"completions/mean_terminated_length": 182.9375,
|
|
"completions/min_length": 123.5,
|
|
"completions/min_terminated_length": 123.5,
|
|
"entropy": 1.094175136089325,
|
|
"epoch": 0.7014801875517146,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.98690938949585,
|
|
"kl": 0.039000157522968946,
|
|
"learning_rate": 4.552314719923073e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 33157824.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.3824188977479935,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.38241890370845794,
|
|
"sampling/importance_sampling_ratio/max": 1.2645862817764282,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000009000301362,
|
|
"sampling/importance_sampling_ratio/min": 0.772594565153122,
|
|
"sampling/sampling_logp_difference/max": 0.28966230154037476,
|
|
"sampling/sampling_logp_difference/mean": 0.02176764216274023,
|
|
"step": 7630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.5,
|
|
"completions/max_terminated_length": 281.5,
|
|
"completions/mean_length": 201.7,
|
|
"completions/mean_terminated_length": 201.7,
|
|
"completions/min_length": 124.5,
|
|
"completions/min_terminated_length": 124.5,
|
|
"entropy": 1.1148397870361806,
|
|
"epoch": 0.7023995587018479,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.051836967468262,
|
|
"kl": 0.039943625044543296,
|
|
"learning_rate": 4.550866878765903e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 33202368.0,
|
|
"reward": -0.00625,
|
|
"reward_std": 0.27346964478492736,
|
|
"rewards/reward_fn/mean": -0.00625,
|
|
"rewards/reward_fn/std": 0.27346965074539187,
|
|
"sampling/importance_sampling_ratio/max": 1.2642905831336975,
|
|
"sampling/importance_sampling_ratio/mean": 1.000321626663208,
|
|
"sampling/importance_sampling_ratio/min": 0.7478184580802918,
|
|
"sampling/sampling_logp_difference/max": 0.3064985632896423,
|
|
"sampling/sampling_logp_difference/mean": 0.021074975468218328,
|
|
"step": 7640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.5,
|
|
"completions/max_terminated_length": 292.5,
|
|
"completions/mean_length": 204.55,
|
|
"completions/mean_terminated_length": 204.55,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 1.09742097184062,
|
|
"epoch": 0.7033189298519812,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.594697952270508,
|
|
"kl": 0.041919013380538675,
|
|
"learning_rate": 4.549416931204051e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 33245476.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.3049629807472229,
|
|
"rewards/reward_fn/mean": 0.4375,
|
|
"rewards/reward_fn/std": 0.3049629867076874,
|
|
"sampling/importance_sampling_ratio/max": 1.2665457367897033,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000653088092804,
|
|
"sampling/importance_sampling_ratio/min": 0.7821203589439392,
|
|
"sampling/sampling_logp_difference/max": 0.26658855676651,
|
|
"sampling/sampling_logp_difference/mean": 0.021193957701325417,
|
|
"step": 7650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.1,
|
|
"completions/max_terminated_length": 286.1,
|
|
"completions/mean_length": 202.2375,
|
|
"completions/mean_terminated_length": 202.2375,
|
|
"completions/min_length": 134.5,
|
|
"completions/min_terminated_length": 134.5,
|
|
"entropy": 1.1349547736346721,
|
|
"epoch": 0.7042383010021146,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.6268391609191895,
|
|
"kl": 0.040196143608773126,
|
|
"learning_rate": 4.547964878726729e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 33287247.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.5108078241348266,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.5108078360557556,
|
|
"sampling/importance_sampling_ratio/max": 1.3117812991142273,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000822067260742,
|
|
"sampling/importance_sampling_ratio/min": 0.7875050127506256,
|
|
"sampling/sampling_logp_difference/max": 0.28344073295593264,
|
|
"sampling/sampling_logp_difference/mean": 0.022066364251077174,
|
|
"step": 7660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.2,
|
|
"completions/max_terminated_length": 258.2,
|
|
"completions/mean_length": 192.1375,
|
|
"completions/mean_terminated_length": 192.1375,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.1447755262255668,
|
|
"epoch": 0.7051576721522479,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.412333965301514,
|
|
"kl": 0.05213334017898887,
|
|
"learning_rate": 4.5465107228253135e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 33327450.0,
|
|
"reward": 0.18125,
|
|
"reward_std": 0.3508347451686859,
|
|
"rewards/reward_fn/mean": 0.18125,
|
|
"rewards/reward_fn/std": 0.3508347511291504,
|
|
"sampling/importance_sampling_ratio/max": 1.2710581183433534,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999002158641815,
|
|
"sampling/importance_sampling_ratio/min": 0.7738643288612366,
|
|
"sampling/sampling_logp_difference/max": 0.29165835976600646,
|
|
"sampling/sampling_logp_difference/mean": 0.022472849115729332,
|
|
"step": 7670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.2,
|
|
"completions/max_terminated_length": 298.2,
|
|
"completions/mean_length": 209.2625,
|
|
"completions/mean_terminated_length": 209.2625,
|
|
"completions/min_length": 145.7,
|
|
"completions/min_terminated_length": 145.7,
|
|
"entropy": 0.9848915636539459,
|
|
"epoch": 0.7060770433023812,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.09794360399246216,
|
|
"kl": 0.0333785520051606,
|
|
"learning_rate": 4.5450544649933375e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 33368159.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.3349648594856262,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.3349648714065552,
|
|
"sampling/importance_sampling_ratio/max": 1.2900812029838562,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998264491558075,
|
|
"sampling/importance_sampling_ratio/min": 0.7895427644252777,
|
|
"sampling/sampling_logp_difference/max": 0.2809017479419708,
|
|
"sampling/sampling_logp_difference/mean": 0.02042638398706913,
|
|
"step": 7680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.6,
|
|
"completions/max_terminated_length": 277.6,
|
|
"completions/mean_length": 198.4,
|
|
"completions/mean_terminated_length": 198.4,
|
|
"completions/min_length": 140.7,
|
|
"completions/min_terminated_length": 140.7,
|
|
"entropy": 1.0890538729727268,
|
|
"epoch": 0.7069964144525145,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.16050505638122559,
|
|
"kl": 0.043837835127487776,
|
|
"learning_rate": 4.543596106726494e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 33408623.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.42010811567306516,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.4201081216335297,
|
|
"sampling/importance_sampling_ratio/max": 1.35472332239151,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003214716911315,
|
|
"sampling/importance_sampling_ratio/min": 0.7823984801769257,
|
|
"sampling/sampling_logp_difference/max": 0.33009341955184934,
|
|
"sampling/sampling_logp_difference/mean": 0.021598795801401137,
|
|
"step": 7690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.6,
|
|
"completions/max_terminated_length": 280.6,
|
|
"completions/mean_length": 202.9625,
|
|
"completions/mean_terminated_length": 202.9625,
|
|
"completions/min_length": 132.3,
|
|
"completions/min_terminated_length": 132.3,
|
|
"entropy": 1.0056064058095218,
|
|
"epoch": 0.7079157856026478,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.50482702255249,
|
|
"kl": 0.04954571602283977,
|
|
"learning_rate": 4.5421356495226356e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 33451972.0,
|
|
"reward": -0.225,
|
|
"reward_std": 0.48828638792037965,
|
|
"rewards/reward_fn/mean": -0.225,
|
|
"rewards/reward_fn/std": 0.4882863938808441,
|
|
"sampling/importance_sampling_ratio/max": 1.271443784236908,
|
|
"sampling/importance_sampling_ratio/mean": 0.9991957306861877,
|
|
"sampling/importance_sampling_ratio/min": 0.7375916063785553,
|
|
"sampling/sampling_logp_difference/max": 0.3386650800704956,
|
|
"sampling/sampling_logp_difference/mean": 0.0205399002879858,
|
|
"step": 7700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.7,
|
|
"completions/max_terminated_length": 297.7,
|
|
"completions/mean_length": 205.3,
|
|
"completions/mean_terminated_length": 205.3,
|
|
"completions/min_length": 135.2,
|
|
"completions/min_terminated_length": 135.2,
|
|
"entropy": 1.1315946891903876,
|
|
"epoch": 0.7088351567527811,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1856188327074051,
|
|
"kl": 0.040633356000762434,
|
|
"learning_rate": 4.540673094881768e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 33494220.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.2966939479112625,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.296693953871727,
|
|
"sampling/importance_sampling_ratio/max": 1.234983253479004,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000481128692627,
|
|
"sampling/importance_sampling_ratio/min": 0.7976975739002228,
|
|
"sampling/sampling_logp_difference/max": 0.23871080875396727,
|
|
"sampling/sampling_logp_difference/mean": 0.02253729533404112,
|
|
"step": 7710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 316.6,
|
|
"completions/max_terminated_length": 302.6,
|
|
"completions/mean_length": 227.625,
|
|
"completions/mean_terminated_length": 224.5,
|
|
"completions/min_length": 151.0,
|
|
"completions/min_terminated_length": 151.0,
|
|
"entropy": 1.1352139122784137,
|
|
"epoch": 0.7097545279029144,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.784977436065674,
|
|
"kl": 0.04954952227417379,
|
|
"learning_rate": 4.539208444306051e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 33540222.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.3108975887298584,
|
|
"rewards/reward_fn/mean": 0.09375,
|
|
"rewards/reward_fn/std": 0.3108976066112518,
|
|
"sampling/importance_sampling_ratio/max": 1.271518075466156,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999210596084595,
|
|
"sampling/importance_sampling_ratio/min": 0.7494085669517517,
|
|
"sampling/sampling_logp_difference/max": 0.306148886680603,
|
|
"sampling/sampling_logp_difference/mean": 0.022181358188390732,
|
|
"step": 7720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.4,
|
|
"completions/max_terminated_length": 315.4,
|
|
"completions/mean_length": 226.4125,
|
|
"completions/mean_terminated_length": 226.4125,
|
|
"completions/min_length": 160.9,
|
|
"completions/min_terminated_length": 160.9,
|
|
"entropy": 1.0955932341516017,
|
|
"epoch": 0.7106738990530477,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.577101230621338,
|
|
"kl": 0.03685650514089502,
|
|
"learning_rate": 4.5377416992998e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 33585631.0,
|
|
"reward": -0.13125,
|
|
"reward_std": 0.38051809966564176,
|
|
"rewards/reward_fn/mean": -0.13125,
|
|
"rewards/reward_fn/std": 0.3805181056261063,
|
|
"sampling/importance_sampling_ratio/max": 1.2924562692642212,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002309203147888,
|
|
"sampling/importance_sampling_ratio/min": 0.7845674693584442,
|
|
"sampling/sampling_logp_difference/max": 0.2886636972427368,
|
|
"sampling/sampling_logp_difference/mean": 0.021633667312562464,
|
|
"step": 7730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.7,
|
|
"completions/max_terminated_length": 282.7,
|
|
"completions/mean_length": 207.9625,
|
|
"completions/mean_terminated_length": 207.9625,
|
|
"completions/min_length": 133.7,
|
|
"completions/min_terminated_length": 133.7,
|
|
"entropy": 1.0437857307493688,
|
|
"epoch": 0.711593270203181,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.8421807289123535,
|
|
"kl": 0.03807074510259554,
|
|
"learning_rate": 4.536272861369478e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 33628524.0,
|
|
"reward": -0.2,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/reward_fn/mean": -0.2,
|
|
"rewards/reward_fn/std": 0.3202547788619995,
|
|
"sampling/importance_sampling_ratio/max": 1.3327083826065063,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998747706413269,
|
|
"sampling/importance_sampling_ratio/min": 0.7640017390251159,
|
|
"sampling/sampling_logp_difference/max": 0.316588830947876,
|
|
"sampling/sampling_logp_difference/mean": 0.021363392099738122,
|
|
"step": 7740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 307.9,
|
|
"completions/max_terminated_length": 285.7,
|
|
"completions/mean_length": 211.275,
|
|
"completions/mean_terminated_length": 207.51428680419923,
|
|
"completions/min_length": 125.6,
|
|
"completions/min_terminated_length": 125.6,
|
|
"entropy": 1.1353821109980344,
|
|
"epoch": 0.7125126413533144,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.20360620319843292,
|
|
"kl": 0.03745500167715363,
|
|
"learning_rate": 4.534801932023701e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 33672482.0,
|
|
"reward": 0.31875,
|
|
"reward_std": 0.3897308617830276,
|
|
"rewards/reward_fn/mean": 0.31875,
|
|
"rewards/reward_fn/std": 0.38973086774349214,
|
|
"sampling/importance_sampling_ratio/max": 1.265360164642334,
|
|
"sampling/importance_sampling_ratio/mean": 1.000350195169449,
|
|
"sampling/importance_sampling_ratio/min": 0.7864132761955261,
|
|
"sampling/sampling_logp_difference/max": 0.2651828408241272,
|
|
"sampling/sampling_logp_difference/mean": 0.022775939479470254,
|
|
"step": 7750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 318.5,
|
|
"completions/max_terminated_length": 306.1,
|
|
"completions/mean_length": 210.2625,
|
|
"completions/mean_terminated_length": 207.0464294433594,
|
|
"completions/min_length": 132.9,
|
|
"completions/min_terminated_length": 132.9,
|
|
"entropy": 1.0103291809558868,
|
|
"epoch": 0.7134320125034477,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.13381251692771912,
|
|
"kl": 0.0372956077917479,
|
|
"learning_rate": 4.53332891277323e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 33715023.0,
|
|
"reward": 0.19375,
|
|
"reward_std": 0.3609550654888153,
|
|
"rewards/reward_fn/mean": 0.19375,
|
|
"rewards/reward_fn/std": 0.36095508337020876,
|
|
"sampling/importance_sampling_ratio/max": 1.2696126341819762,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006945371627807,
|
|
"sampling/importance_sampling_ratio/min": 0.7741854786872864,
|
|
"sampling/sampling_logp_difference/max": 0.2791248619556427,
|
|
"sampling/sampling_logp_difference/mean": 0.020971699617803097,
|
|
"step": 7760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.1,
|
|
"completions/max_terminated_length": 319.1,
|
|
"completions/mean_length": 224.025,
|
|
"completions/mean_terminated_length": 224.025,
|
|
"completions/min_length": 159.2,
|
|
"completions/min_terminated_length": 159.2,
|
|
"entropy": 1.1375169530510902,
|
|
"epoch": 0.714351383653581,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12078677117824554,
|
|
"kl": 0.037285058957058935,
|
|
"learning_rate": 4.531853805130975e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 33761481.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.3038551896810532,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.30385519564151764,
|
|
"sampling/importance_sampling_ratio/max": 1.221822166442871,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995638489723205,
|
|
"sampling/importance_sampling_ratio/min": 0.7730231702327728,
|
|
"sampling/sampling_logp_difference/max": 0.2626751184463501,
|
|
"sampling/sampling_logp_difference/mean": 0.022068896889686586,
|
|
"step": 7770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 252.0,
|
|
"completions/max_terminated_length": 252.0,
|
|
"completions/mean_length": 185.9625,
|
|
"completions/mean_terminated_length": 185.9625,
|
|
"completions/min_length": 128.6,
|
|
"completions/min_terminated_length": 128.6,
|
|
"entropy": 1.0630664438009263,
|
|
"epoch": 0.7152707548037143,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.958263874053955,
|
|
"kl": 0.031935215438716115,
|
|
"learning_rate": 4.530376610611988e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 33801398.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.2877043843269348,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.28770439624786376,
|
|
"sampling/importance_sampling_ratio/max": 1.2999430894851685,
|
|
"sampling/importance_sampling_ratio/mean": 1.000385844707489,
|
|
"sampling/importance_sampling_ratio/min": 0.7916532039642334,
|
|
"sampling/sampling_logp_difference/max": 0.27481067180633545,
|
|
"sampling/sampling_logp_difference/mean": 0.02188342474400997,
|
|
"step": 7780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.6,
|
|
"completions/max_terminated_length": 309.6,
|
|
"completions/mean_length": 211.5875,
|
|
"completions/mean_terminated_length": 211.5875,
|
|
"completions/min_length": 139.5,
|
|
"completions/min_terminated_length": 139.5,
|
|
"entropy": 1.0215931819751858,
|
|
"epoch": 0.7161901259538476,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.65397310256958,
|
|
"kl": 0.039828912253142335,
|
|
"learning_rate": 4.528897330733467e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 33845397.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.44539301097393036,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.44539301097393036,
|
|
"sampling/importance_sampling_ratio/max": 1.2443268537521361,
|
|
"sampling/importance_sampling_ratio/mean": 0.999433183670044,
|
|
"sampling/importance_sampling_ratio/min": 0.806364780664444,
|
|
"sampling/sampling_logp_difference/max": 0.24003589153289795,
|
|
"sampling/sampling_logp_difference/mean": 0.019792910479009152,
|
|
"step": 7790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.4,
|
|
"completions/max_terminated_length": 261.4,
|
|
"completions/mean_length": 190.15,
|
|
"completions/mean_terminated_length": 190.15,
|
|
"completions/min_length": 135.3,
|
|
"completions/min_terminated_length": 135.3,
|
|
"entropy": 0.9830567568540574,
|
|
"epoch": 0.7171094971039809,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.937559604644775,
|
|
"kl": 0.0322166383266449,
|
|
"learning_rate": 4.527415967014752e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 33887569.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.3504155069589615,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.35041551291942596,
|
|
"sampling/importance_sampling_ratio/max": 1.2904535412788392,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998431980609894,
|
|
"sampling/importance_sampling_ratio/min": 0.7544145703315734,
|
|
"sampling/sampling_logp_difference/max": 0.324628746509552,
|
|
"sampling/sampling_logp_difference/mean": 0.020097553543746472,
|
|
"step": 7800
|
|
},
|
|
{
|
|
"epoch": 0.7171094971039809,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002399650959860384,
|
|
"eval_completions/max_length": 307.7190226876091,
|
|
"eval_completions/max_terminated_length": 305.66841186736474,
|
|
"eval_completions/mean_length": 219.8651832460733,
|
|
"eval_completions/mean_terminated_length": 219.30843325310352,
|
|
"eval_completions/min_length": 146.96684118673647,
|
|
"eval_completions/min_terminated_length": 146.96684118673647,
|
|
"eval_entropy": 1.0932212774666192,
|
|
"eval_frac_reward_zero_std": 0.30715532286212915,
|
|
"eval_kl": 0.042738341207877714,
|
|
"eval_loss": 0.0021742191165685654,
|
|
"eval_num_tokens": 33887569.0,
|
|
"eval_reward": 0.05279232111692845,
|
|
"eval_reward_std": 0.3492573535047068,
|
|
"eval_rewards/reward_fn/mean": 0.05279232111692845,
|
|
"eval_rewards/reward_fn/std": 0.3492573633868033,
|
|
"eval_runtime": 1787.3842,
|
|
"eval_samples_per_second": 0.321,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2752930914128222,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999584233157506,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7807589556638275,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27663113673528034,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020761492777531268,
|
|
"eval_steps_per_second": 0.04,
|
|
"step": 7800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 342.1,
|
|
"completions/max_terminated_length": 342.1,
|
|
"completions/mean_length": 242.25,
|
|
"completions/mean_terminated_length": 242.25,
|
|
"completions/min_length": 161.1,
|
|
"completions/min_terminated_length": 161.1,
|
|
"entropy": 1.139425777643919,
|
|
"epoch": 0.7180288682541142,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.486752510070801,
|
|
"kl": 0.05300962534965947,
|
|
"learning_rate": 4.525932520977322e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 33937949.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.404693940281868,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.404693940281868,
|
|
"sampling/importance_sampling_ratio/max": 1.2692065238952637,
|
|
"sampling/importance_sampling_ratio/mean": 1.000042188167572,
|
|
"sampling/importance_sampling_ratio/min": 0.7585910797119141,
|
|
"sampling/sampling_logp_difference/max": 0.2811714351177216,
|
|
"sampling/sampling_logp_difference/mean": 0.021924970485270024,
|
|
"step": 7810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.7,
|
|
"completions/max_terminated_length": 281.7,
|
|
"completions/mean_length": 198.9375,
|
|
"completions/mean_terminated_length": 198.9375,
|
|
"completions/min_length": 144.8,
|
|
"completions/min_terminated_length": 144.8,
|
|
"entropy": 1.0535544984042644,
|
|
"epoch": 0.7189482394042475,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12694041430950165,
|
|
"kl": 0.046225569950183856,
|
|
"learning_rate": 4.524446994144795e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 33981160.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.361081862449646,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.3610818684101105,
|
|
"sampling/importance_sampling_ratio/max": 1.252843940258026,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002496123313904,
|
|
"sampling/importance_sampling_ratio/min": 0.7583630740642547,
|
|
"sampling/sampling_logp_difference/max": 0.28971938490867616,
|
|
"sampling/sampling_logp_difference/mean": 0.021731074154376983,
|
|
"step": 7820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.8,
|
|
"completions/max_terminated_length": 295.8,
|
|
"completions/mean_length": 216.9625,
|
|
"completions/mean_terminated_length": 216.9625,
|
|
"completions/min_length": 152.7,
|
|
"completions/min_terminated_length": 152.7,
|
|
"entropy": 1.0459695369005204,
|
|
"epoch": 0.7198676105543808,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.909083366394043,
|
|
"kl": 0.042244491196470335,
|
|
"learning_rate": 4.5229593880429245e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 34026213.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.38372147977352145,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.38372149765491487,
|
|
"sampling/importance_sampling_ratio/max": 1.2656212329864502,
|
|
"sampling/importance_sampling_ratio/mean": 0.999660336971283,
|
|
"sampling/importance_sampling_ratio/min": 0.7756404161453248,
|
|
"sampling/sampling_logp_difference/max": 0.275792670249939,
|
|
"sampling/sampling_logp_difference/mean": 0.021000822819769383,
|
|
"step": 7830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.6,
|
|
"completions/max_terminated_length": 325.6,
|
|
"completions/mean_length": 233.5625,
|
|
"completions/mean_terminated_length": 233.5625,
|
|
"completions/min_length": 146.6,
|
|
"completions/min_terminated_length": 146.6,
|
|
"entropy": 1.1200813628733157,
|
|
"epoch": 0.7207869817045142,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.11171316355466843,
|
|
"kl": 0.047881312703248115,
|
|
"learning_rate": 4.521469704199603e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 34071514.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.25032021999359133,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.25032023191452024,
|
|
"sampling/importance_sampling_ratio/max": 1.3082908630371093,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002647995948792,
|
|
"sampling/importance_sampling_ratio/min": 0.7753558576107025,
|
|
"sampling/sampling_logp_difference/max": 0.2913394570350647,
|
|
"sampling/sampling_logp_difference/mean": 0.021616406552493573,
|
|
"step": 7840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.8,
|
|
"completions/max_terminated_length": 308.8,
|
|
"completions/mean_length": 214.175,
|
|
"completions/mean_terminated_length": 214.175,
|
|
"completions/min_length": 153.9,
|
|
"completions/min_terminated_length": 153.9,
|
|
"entropy": 1.0277395989745854,
|
|
"epoch": 0.7217063528546475,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 4.850182056427002,
|
|
"kl": 0.052500088210217656,
|
|
"learning_rate": 4.519977944144854e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 34114304.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.4219646632671356,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.4219646692276001,
|
|
"sampling/importance_sampling_ratio/max": 1.2588640332221985,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002581179141998,
|
|
"sampling/importance_sampling_ratio/min": 0.7745973169803619,
|
|
"sampling/sampling_logp_difference/max": 0.2695298671722412,
|
|
"sampling/sampling_logp_difference/mean": 0.02054786393418908,
|
|
"step": 7850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.3,
|
|
"completions/max_terminated_length": 303.3,
|
|
"completions/mean_length": 214.575,
|
|
"completions/mean_terminated_length": 214.575,
|
|
"completions/min_length": 148.5,
|
|
"completions/min_terminated_length": 148.5,
|
|
"entropy": 1.1137353211641312,
|
|
"epoch": 0.7226257240047808,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.13138894736766815,
|
|
"kl": 0.050362446304643524,
|
|
"learning_rate": 4.518484109410837e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 34158102.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.38927595913410185,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.38927596509456636,
|
|
"sampling/importance_sampling_ratio/max": 1.262539768218994,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997757256031037,
|
|
"sampling/importance_sampling_ratio/min": 0.8053169488906861,
|
|
"sampling/sampling_logp_difference/max": 0.24541983604431153,
|
|
"sampling/sampling_logp_difference/mean": 0.021543565765023233,
|
|
"step": 7860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.9,
|
|
"completions/max_terminated_length": 309.9,
|
|
"completions/mean_length": 212.8125,
|
|
"completions/mean_terminated_length": 212.8125,
|
|
"completions/min_length": 139.8,
|
|
"completions/min_terminated_length": 139.8,
|
|
"entropy": 1.0954293794929981,
|
|
"epoch": 0.723545095154914,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1322997808456421,
|
|
"kl": 0.03421082017011941,
|
|
"learning_rate": 4.516988201531837e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 34202567.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.3918117702007294,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.3918117702007294,
|
|
"sampling/importance_sampling_ratio/max": 1.2883691906929016,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999663412570954,
|
|
"sampling/importance_sampling_ratio/min": 0.7708441138267517,
|
|
"sampling/sampling_logp_difference/max": 0.31276854276657107,
|
|
"sampling/sampling_logp_difference/mean": 0.021195939183235167,
|
|
"step": 7870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.2,
|
|
"completions/max_terminated_length": 297.2,
|
|
"completions/mean_length": 219.45,
|
|
"completions/mean_terminated_length": 219.45,
|
|
"completions/min_length": 152.1,
|
|
"completions/min_terminated_length": 152.1,
|
|
"entropy": 1.072926290333271,
|
|
"epoch": 0.7244644663050473,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.371822834014893,
|
|
"kl": 0.04007494014222175,
|
|
"learning_rate": 4.515490222044273e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 34249091.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.2857582807540894,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.28575828671455383,
|
|
"sampling/importance_sampling_ratio/max": 1.2246232509613038,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997083663940429,
|
|
"sampling/importance_sampling_ratio/min": 0.7805475413799285,
|
|
"sampling/sampling_logp_difference/max": 0.25602915287017824,
|
|
"sampling/sampling_logp_difference/mean": 0.020623759552836417,
|
|
"step": 7880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.1,
|
|
"completions/max_terminated_length": 289.1,
|
|
"completions/mean_length": 208.4875,
|
|
"completions/mean_terminated_length": 208.4875,
|
|
"completions/min_length": 135.0,
|
|
"completions/min_terminated_length": 135.0,
|
|
"entropy": 1.0243792064487933,
|
|
"epoch": 0.7253838374551806,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.473594665527344,
|
|
"kl": 0.04635031825164333,
|
|
"learning_rate": 4.513990172486689e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 34292122.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.23144718110561371,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.23144719302654265,
|
|
"sampling/importance_sampling_ratio/max": 1.30718754529953,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002975821495057,
|
|
"sampling/importance_sampling_ratio/min": 0.7781687617301941,
|
|
"sampling/sampling_logp_difference/max": 0.29209108352661134,
|
|
"sampling/sampling_logp_difference/mean": 0.021396419778466225,
|
|
"step": 7890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 337.1,
|
|
"completions/max_terminated_length": 337.1,
|
|
"completions/mean_length": 219.1,
|
|
"completions/mean_terminated_length": 219.1,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 1.0985589452087878,
|
|
"epoch": 0.7263032086053139,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.08928769826889038,
|
|
"kl": 0.04869061856297776,
|
|
"learning_rate": 4.512488054399758e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 34338882.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.34927949607372283,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.34927951991558076,
|
|
"sampling/importance_sampling_ratio/max": 1.2863481521606446,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998274266719818,
|
|
"sampling/importance_sampling_ratio/min": 0.7827123999595642,
|
|
"sampling/sampling_logp_difference/max": 0.2661070704460144,
|
|
"sampling/sampling_logp_difference/mean": 0.021367667987942696,
|
|
"step": 7900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.2,
|
|
"completions/max_terminated_length": 288.2,
|
|
"completions/mean_length": 197.4375,
|
|
"completions/mean_terminated_length": 197.4375,
|
|
"completions/min_length": 128.8,
|
|
"completions/min_terminated_length": 128.8,
|
|
"entropy": 1.0174917988479137,
|
|
"epoch": 0.7272225797554472,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1163519099354744,
|
|
"kl": 0.04199837712221779,
|
|
"learning_rate": 4.510983869326273e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 34379477.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.42832300066947937,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.42832301259040834,
|
|
"sampling/importance_sampling_ratio/max": 1.2799532771110536,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994352340698243,
|
|
"sampling/importance_sampling_ratio/min": 0.7756819188594818,
|
|
"sampling/sampling_logp_difference/max": 0.28632594347000123,
|
|
"sampling/sampling_logp_difference/mean": 0.02054649069905281,
|
|
"step": 7910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.8,
|
|
"completions/max_terminated_length": 275.8,
|
|
"completions/mean_length": 191.9,
|
|
"completions/mean_terminated_length": 191.9,
|
|
"completions/min_length": 128.8,
|
|
"completions/min_terminated_length": 128.8,
|
|
"entropy": 1.0416158333420753,
|
|
"epoch": 0.7281419509055805,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.041605472564697,
|
|
"kl": 0.03358384228777141,
|
|
"learning_rate": 4.509477618811156e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 34421453.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.4782218664884567,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.4782218724489212,
|
|
"sampling/importance_sampling_ratio/max": 1.2627132177352904,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999583303928375,
|
|
"sampling/importance_sampling_ratio/min": 0.7706750512123108,
|
|
"sampling/sampling_logp_difference/max": 0.275869745016098,
|
|
"sampling/sampling_logp_difference/mean": 0.02102560121566057,
|
|
"step": 7920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 324.7,
|
|
"completions/max_terminated_length": 319.2,
|
|
"completions/mean_length": 217.325,
|
|
"completions/mean_terminated_length": 214.61964416503906,
|
|
"completions/min_length": 143.1,
|
|
"completions/min_terminated_length": 143.1,
|
|
"entropy": 1.090885204076767,
|
|
"epoch": 0.729061322055714,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.933417797088623,
|
|
"kl": 0.031383454659953715,
|
|
"learning_rate": 4.507969304401443e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 34466215.0,
|
|
"reward": 0.06875,
|
|
"reward_std": 0.285294908285141,
|
|
"rewards/reward_fn/mean": 0.06875,
|
|
"rewards/reward_fn/std": 0.285294908285141,
|
|
"sampling/importance_sampling_ratio/max": 1.285138475894928,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998807489871979,
|
|
"sampling/importance_sampling_ratio/min": 0.7618761599063874,
|
|
"sampling/sampling_logp_difference/max": 0.3024583339691162,
|
|
"sampling/sampling_logp_difference/mean": 0.021107158064842223,
|
|
"step": 7930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.1,
|
|
"completions/max_terminated_length": 283.1,
|
|
"completions/mean_length": 207.875,
|
|
"completions/mean_terminated_length": 207.875,
|
|
"completions/min_length": 137.4,
|
|
"completions/min_terminated_length": 137.4,
|
|
"entropy": 0.9704998046159744,
|
|
"epoch": 0.7299806932058472,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12281299382448196,
|
|
"kl": 0.030406964721623807,
|
|
"learning_rate": 4.506458927646297e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 34506877.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.26849985122680664,
|
|
"rewards/reward_fn/mean": 0.3875,
|
|
"rewards/reward_fn/std": 0.2684998571872711,
|
|
"sampling/importance_sampling_ratio/max": 1.2835866928100585,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997747421264649,
|
|
"sampling/importance_sampling_ratio/min": 0.7811548173427582,
|
|
"sampling/sampling_logp_difference/max": 0.28658034801483157,
|
|
"sampling/sampling_logp_difference/mean": 0.020425513759255408,
|
|
"step": 7940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 343.8,
|
|
"completions/max_terminated_length": 343.8,
|
|
"completions/mean_length": 230.8875,
|
|
"completions/mean_terminated_length": 230.8875,
|
|
"completions/min_length": 147.4,
|
|
"completions/min_terminated_length": 147.4,
|
|
"entropy": 1.1622434698045254,
|
|
"epoch": 0.7309000643559805,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.867983341217041,
|
|
"kl": 0.03975532315671444,
|
|
"learning_rate": 4.504946490096995e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 34549692.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.340069380402565,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.340069380402565,
|
|
"sampling/importance_sampling_ratio/max": 1.280290699005127,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001557052135468,
|
|
"sampling/importance_sampling_ratio/min": 0.7522724390029907,
|
|
"sampling/sampling_logp_difference/max": 0.32168265283107755,
|
|
"sampling/sampling_logp_difference/mean": 0.022287780605256557,
|
|
"step": 7950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 328.6,
|
|
"completions/max_terminated_length": 328.6,
|
|
"completions/mean_length": 212.525,
|
|
"completions/mean_terminated_length": 212.525,
|
|
"completions/min_length": 134.5,
|
|
"completions/min_terminated_length": 134.5,
|
|
"entropy": 1.0840055771172046,
|
|
"epoch": 0.7318194355061138,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.8784050941467285,
|
|
"kl": 0.05069787713000551,
|
|
"learning_rate": 4.503431993306931e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 34591006.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.5297319799661636,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.5297319859266281,
|
|
"sampling/importance_sampling_ratio/max": 1.2535661220550538,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000344336032867,
|
|
"sampling/importance_sampling_ratio/min": 0.805215185880661,
|
|
"sampling/sampling_logp_difference/max": 0.2535577893257141,
|
|
"sampling/sampling_logp_difference/mean": 0.02068595476448536,
|
|
"step": 7960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.4,
|
|
"completions/max_terminated_length": 295.4,
|
|
"completions/mean_length": 214.95,
|
|
"completions/mean_terminated_length": 214.95,
|
|
"completions/min_length": 147.8,
|
|
"completions/min_terminated_length": 147.8,
|
|
"entropy": 1.1212335646152496,
|
|
"epoch": 0.7327388066562471,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.346487522125244,
|
|
"kl": 0.04603611533530057,
|
|
"learning_rate": 4.5019154388316165e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 34636938.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.2937580019235611,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.29375800788402556,
|
|
"sampling/importance_sampling_ratio/max": 1.2788280606269837,
|
|
"sampling/importance_sampling_ratio/mean": 1.000061458349228,
|
|
"sampling/importance_sampling_ratio/min": 0.7785823583602905,
|
|
"sampling/sampling_logp_difference/max": 0.27463321685791015,
|
|
"sampling/sampling_logp_difference/mean": 0.021402878873050212,
|
|
"step": 7970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 299.1,
|
|
"completions/max_terminated_length": 275.4,
|
|
"completions/mean_length": 202.575,
|
|
"completions/mean_terminated_length": 198.7732147216797,
|
|
"completions/min_length": 131.7,
|
|
"completions/min_terminated_length": 131.7,
|
|
"entropy": 1.0417906403541566,
|
|
"epoch": 0.7336581778063804,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 8.332976341247559,
|
|
"kl": 0.044533792143920434,
|
|
"learning_rate": 4.500396828228673e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 34681400.0,
|
|
"reward": 0.14375,
|
|
"reward_std": 0.34977678656578065,
|
|
"rewards/reward_fn/mean": 0.14375,
|
|
"rewards/reward_fn/std": 0.34977678656578065,
|
|
"sampling/importance_sampling_ratio/max": 1.2818006515502929,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997791945934296,
|
|
"sampling/importance_sampling_ratio/min": 0.732440397143364,
|
|
"sampling/sampling_logp_difference/max": 0.3559589147567749,
|
|
"sampling/sampling_logp_difference/mean": 0.020886536128818987,
|
|
"step": 7980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.8,
|
|
"completions/max_terminated_length": 287.8,
|
|
"completions/mean_length": 198.0375,
|
|
"completions/mean_terminated_length": 198.0375,
|
|
"completions/min_length": 132.2,
|
|
"completions/min_terminated_length": 132.2,
|
|
"entropy": 1.085494488477707,
|
|
"epoch": 0.7345775489565137,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.626796722412109,
|
|
"kl": 0.026610825723037124,
|
|
"learning_rate": 4.498876163057834e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 34725099.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.29642280638217927,
|
|
"rewards/reward_fn/mean": 0.375,
|
|
"rewards/reward_fn/std": 0.29642280638217927,
|
|
"sampling/importance_sampling_ratio/max": 1.2888177275657653,
|
|
"sampling/importance_sampling_ratio/mean": 1.000968623161316,
|
|
"sampling/importance_sampling_ratio/min": 0.7828558683395386,
|
|
"sampling/sampling_logp_difference/max": 0.26327439546585085,
|
|
"sampling/sampling_logp_difference/mean": 0.021558471396565436,
|
|
"step": 7990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.4,
|
|
"completions/max_terminated_length": 280.4,
|
|
"completions/mean_length": 193.5125,
|
|
"completions/mean_terminated_length": 193.5125,
|
|
"completions/min_length": 128.6,
|
|
"completions/min_terminated_length": 128.6,
|
|
"entropy": 1.0552413880825042,
|
|
"epoch": 0.735496920106647,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.10022339224815369,
|
|
"kl": 0.053475710132624955,
|
|
"learning_rate": 4.497353444880946e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 34764924.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.35827490985393523,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.35827491581439974,
|
|
"sampling/importance_sampling_ratio/max": 1.2680016875267028,
|
|
"sampling/importance_sampling_ratio/mean": 0.999761426448822,
|
|
"sampling/importance_sampling_ratio/min": 0.7820928394794464,
|
|
"sampling/sampling_logp_difference/max": 0.277970677614212,
|
|
"sampling/sampling_logp_difference/mean": 0.02173848431557417,
|
|
"step": 8000
|
|
},
|
|
{
|
|
"epoch": 0.735496920106647,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0010907504363001745,
|
|
"eval_completions/max_length": 283.7993019197208,
|
|
"eval_completions/max_terminated_length": 283.2547993019197,
|
|
"eval_completions/mean_length": 197.25501745200697,
|
|
"eval_completions/mean_terminated_length": 197.0399526759176,
|
|
"eval_completions/min_length": 126.37347294938918,
|
|
"eval_completions/min_terminated_length": 126.37347294938918,
|
|
"eval_entropy": 1.0924774579456755,
|
|
"eval_frac_reward_zero_std": 0.29668411867364747,
|
|
"eval_kl": 0.0382567643696673,
|
|
"eval_loss": 0.001952486694790423,
|
|
"eval_num_tokens": 34764924.0,
|
|
"eval_reward": 0.03468586387434555,
|
|
"eval_reward_std": 0.37787579387478387,
|
|
"eval_rewards/reward_fn/mean": 0.03468586387434555,
|
|
"eval_rewards/reward_fn/std": 0.37787580521318925,
|
|
"eval_runtime": 1665.7685,
|
|
"eval_samples_per_second": 0.344,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2756718395060063,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999773400408227,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7776078785173139,
|
|
"eval_sampling/sampling_logp_difference/max": 0.28330271042246263,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021186423596859395,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 8000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.6,
|
|
"completions/max_terminated_length": 274.6,
|
|
"completions/mean_length": 189.9625,
|
|
"completions/mean_terminated_length": 189.9625,
|
|
"completions/min_length": 117.9,
|
|
"completions/min_terminated_length": 117.9,
|
|
"entropy": 1.1164085283875465,
|
|
"epoch": 0.7364162912567803,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.712962627410889,
|
|
"kl": 0.03859089035540819,
|
|
"learning_rate": 4.495828675261964e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 34809193.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.31875187158584595,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.3187518835067749,
|
|
"sampling/importance_sampling_ratio/max": 1.226724374294281,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996237277984619,
|
|
"sampling/importance_sampling_ratio/min": 0.7882050752639771,
|
|
"sampling/sampling_logp_difference/max": 0.2427889585494995,
|
|
"sampling/sampling_logp_difference/mean": 0.021147720888257025,
|
|
"step": 8010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.5,
|
|
"completions/max_terminated_length": 287.5,
|
|
"completions/mean_length": 185.85,
|
|
"completions/mean_terminated_length": 185.85,
|
|
"completions/min_length": 99.5,
|
|
"completions/min_terminated_length": 99.5,
|
|
"entropy": 1.1098272435367107,
|
|
"epoch": 0.7373356624069136,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.14185519516468048,
|
|
"kl": 0.053213424660498276,
|
|
"learning_rate": 4.494301855766947e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 34849221.0,
|
|
"reward": -0.01875,
|
|
"reward_std": 0.25121486783027647,
|
|
"rewards/reward_fn/mean": -0.01875,
|
|
"rewards/reward_fn/std": 0.25121487975120543,
|
|
"sampling/importance_sampling_ratio/max": 1.2421876192092896,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999228894710541,
|
|
"sampling/importance_sampling_ratio/min": 0.7679965972900391,
|
|
"sampling/sampling_logp_difference/max": 0.280995237827301,
|
|
"sampling/sampling_logp_difference/mean": 0.0220765283331275,
|
|
"step": 8020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 327.9,
|
|
"completions/max_terminated_length": 327.9,
|
|
"completions/mean_length": 221.9,
|
|
"completions/mean_terminated_length": 221.9,
|
|
"completions/min_length": 132.5,
|
|
"completions/min_terminated_length": 132.5,
|
|
"entropy": 1.1154138021171094,
|
|
"epoch": 0.738255033557047,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.695866584777832,
|
|
"kl": 0.04086888821911998,
|
|
"learning_rate": 4.492772987964061e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 34890997.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.4650593549013138,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.4650593727827072,
|
|
"sampling/importance_sampling_ratio/max": 1.3206546783447266,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002088248729706,
|
|
"sampling/importance_sampling_ratio/min": 0.7338766843080521,
|
|
"sampling/sampling_logp_difference/max": 0.3303542137145996,
|
|
"sampling/sampling_logp_difference/mean": 0.02125283405184746,
|
|
"step": 8030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.1,
|
|
"completions/max_terminated_length": 302.1,
|
|
"completions/mean_length": 205.1625,
|
|
"completions/mean_terminated_length": 205.1625,
|
|
"completions/min_length": 139.4,
|
|
"completions/min_terminated_length": 139.4,
|
|
"entropy": 1.1147445015609265,
|
|
"epoch": 0.7391744047071803,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.414944648742676,
|
|
"kl": 0.04698902390664443,
|
|
"learning_rate": 4.491242073423575e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 34934050.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.4944003999233246,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.494400417804718,
|
|
"sampling/importance_sampling_ratio/max": 1.3245744705200195,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002626717090606,
|
|
"sampling/importance_sampling_ratio/min": 0.7732381999492646,
|
|
"sampling/sampling_logp_difference/max": 0.31245502829551697,
|
|
"sampling/sampling_logp_difference/mean": 0.02212924640625715,
|
|
"step": 8040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.5,
|
|
"completions/max_terminated_length": 309.5,
|
|
"completions/mean_length": 216.9,
|
|
"completions/mean_terminated_length": 216.9,
|
|
"completions/min_length": 153.9,
|
|
"completions/min_terminated_length": 153.9,
|
|
"entropy": 1.084584641456604,
|
|
"epoch": 0.7400937758573136,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.194960117340088,
|
|
"kl": 0.045161233912222085,
|
|
"learning_rate": 4.489709113717864e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 34977570.0,
|
|
"reward": -0.3375,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/reward_fn/mean": -0.3375,
|
|
"rewards/reward_fn/std": 0.25586686432361605,
|
|
"sampling/importance_sampling_ratio/max": 1.263705599308014,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999563097953796,
|
|
"sampling/importance_sampling_ratio/min": 0.784296065568924,
|
|
"sampling/sampling_logp_difference/max": 0.2550477385520935,
|
|
"sampling/sampling_logp_difference/mean": 0.02131497785449028,
|
|
"step": 8050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.2,
|
|
"completions/max_terminated_length": 314.2,
|
|
"completions/mean_length": 222.075,
|
|
"completions/mean_terminated_length": 222.075,
|
|
"completions/min_length": 137.5,
|
|
"completions/min_terminated_length": 137.5,
|
|
"entropy": 1.117838954180479,
|
|
"epoch": 0.7410131470074469,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.2061675488948822,
|
|
"kl": 0.04983030980220064,
|
|
"learning_rate": 4.488174110421397e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 35023424.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.3746744871139526,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.3746744990348816,
|
|
"sampling/importance_sampling_ratio/max": 1.245806872844696,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993916511535644,
|
|
"sampling/importance_sampling_ratio/min": 0.778225439786911,
|
|
"sampling/sampling_logp_difference/max": 0.2540948510169983,
|
|
"sampling/sampling_logp_difference/mean": 0.021714299358427523,
|
|
"step": 8060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.0,
|
|
"completions/max_terminated_length": 287.0,
|
|
"completions/mean_length": 203.0375,
|
|
"completions/mean_terminated_length": 203.0375,
|
|
"completions/min_length": 143.9,
|
|
"completions/min_terminated_length": 143.9,
|
|
"entropy": 1.1539097726345062,
|
|
"epoch": 0.7419325181575802,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1783839911222458,
|
|
"kl": 0.03680320898420177,
|
|
"learning_rate": 4.4866370651107475e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 35065795.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.32402919232845306,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.32402920424938203,
|
|
"sampling/importance_sampling_ratio/max": 1.2686956644058227,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004583716392517,
|
|
"sampling/importance_sampling_ratio/min": 0.7861164212226868,
|
|
"sampling/sampling_logp_difference/max": 0.27084324359893797,
|
|
"sampling/sampling_logp_difference/mean": 0.02113088108599186,
|
|
"step": 8070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.8,
|
|
"completions/max_terminated_length": 313.8,
|
|
"completions/mean_length": 226.0875,
|
|
"completions/mean_terminated_length": 226.0875,
|
|
"completions/min_length": 148.1,
|
|
"completions/min_terminated_length": 148.1,
|
|
"entropy": 1.1743130065500735,
|
|
"epoch": 0.7428518893077135,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.1916358768939972,
|
|
"kl": 0.04504839995643124,
|
|
"learning_rate": 4.485097979364583e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 35113226.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.40190111100673676,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.4019011229276657,
|
|
"sampling/importance_sampling_ratio/max": 1.2551014065742492,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998955011367798,
|
|
"sampling/importance_sampling_ratio/min": 0.7922015309333801,
|
|
"sampling/sampling_logp_difference/max": 0.2449549913406372,
|
|
"sampling/sampling_logp_difference/mean": 0.02202475965023041,
|
|
"step": 8080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 353.6,
|
|
"completions/max_terminated_length": 353.6,
|
|
"completions/mean_length": 236.7125,
|
|
"completions/mean_terminated_length": 236.7125,
|
|
"completions/min_length": 149.1,
|
|
"completions/min_terminated_length": 149.1,
|
|
"entropy": 1.1358940482139588,
|
|
"epoch": 0.7437712604578468,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.6515960693359375,
|
|
"kl": 0.04084088265663013,
|
|
"learning_rate": 4.48355685476367e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 35159739.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.31855743527412417,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.3185574531555176,
|
|
"sampling/importance_sampling_ratio/max": 1.2688104152679442,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995947360992432,
|
|
"sampling/importance_sampling_ratio/min": 0.7758252382278442,
|
|
"sampling/sampling_logp_difference/max": 0.2809697389602661,
|
|
"sampling/sampling_logp_difference/mean": 0.02172373775392771,
|
|
"step": 8090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.0,
|
|
"completions/max_terminated_length": 274.0,
|
|
"completions/mean_length": 196.1375,
|
|
"completions/mean_terminated_length": 196.1375,
|
|
"completions/min_length": 125.5,
|
|
"completions/min_terminated_length": 125.5,
|
|
"entropy": 1.01872980594635,
|
|
"epoch": 0.7446906316079801,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.545493125915527,
|
|
"kl": 0.05117638137890026,
|
|
"learning_rate": 4.482013692890864e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 35199878.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.4064438194036484,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.4064438432455063,
|
|
"sampling/importance_sampling_ratio/max": 1.2991944909095765,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996060609817505,
|
|
"sampling/importance_sampling_ratio/min": 0.7771979153156281,
|
|
"sampling/sampling_logp_difference/max": 0.27851874828338624,
|
|
"sampling/sampling_logp_difference/mean": 0.020979069732129575,
|
|
"step": 8100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 328.7,
|
|
"completions/max_terminated_length": 310.4,
|
|
"completions/mean_length": 227.5375,
|
|
"completions/mean_terminated_length": 223.9232147216797,
|
|
"completions/min_length": 148.5,
|
|
"completions/min_terminated_length": 148.5,
|
|
"entropy": 1.1326582320034504,
|
|
"epoch": 0.7456100027581134,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 8.607165336608887,
|
|
"kl": 0.0513146938290447,
|
|
"learning_rate": 4.480468495331118e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 35246241.0,
|
|
"reward": -0.08125,
|
|
"reward_std": 0.4195474088191986,
|
|
"rewards/reward_fn/mean": -0.08125,
|
|
"rewards/reward_fn/std": 0.4195474207401276,
|
|
"sampling/importance_sampling_ratio/max": 1.2275462746620178,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999100089073181,
|
|
"sampling/importance_sampling_ratio/min": 0.7939382374286652,
|
|
"sampling/sampling_logp_difference/max": 0.23643859028816222,
|
|
"sampling/sampling_logp_difference/mean": 0.021156668290495874,
|
|
"step": 8110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.6,
|
|
"completions/max_terminated_length": 288.6,
|
|
"completions/mean_length": 210.7375,
|
|
"completions/mean_terminated_length": 210.7375,
|
|
"completions/min_length": 138.7,
|
|
"completions/min_terminated_length": 138.7,
|
|
"entropy": 1.0426415480673312,
|
|
"epoch": 0.7465293739082468,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14802688360214233,
|
|
"kl": 0.057675226614810525,
|
|
"learning_rate": 4.4789212636714745e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 35290980.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.3502548158168793,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.35025482773780825,
|
|
"sampling/importance_sampling_ratio/max": 1.2510496973991394,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000108242034913,
|
|
"sampling/importance_sampling_ratio/min": 0.7605468928813934,
|
|
"sampling/sampling_logp_difference/max": 0.2885816752910614,
|
|
"sampling/sampling_logp_difference/mean": 0.020778967626392843,
|
|
"step": 8120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 320.4,
|
|
"completions/max_terminated_length": 314.9,
|
|
"completions/mean_length": 225.5125,
|
|
"completions/mean_terminated_length": 223.22678833007814,
|
|
"completions/min_length": 151.6,
|
|
"completions/min_terminated_length": 151.6,
|
|
"entropy": 1.0744621112942696,
|
|
"epoch": 0.7474487450583801,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13565051555633545,
|
|
"kl": 0.0431557557429187,
|
|
"learning_rate": 4.477371999501063e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 35338181.0,
|
|
"reward": 0.25625,
|
|
"reward_std": 0.3614010751247406,
|
|
"rewards/reward_fn/mean": 0.25625,
|
|
"rewards/reward_fn/std": 0.36140108704566953,
|
|
"sampling/importance_sampling_ratio/max": 1.2702150106430055,
|
|
"sampling/importance_sampling_ratio/mean": 0.999653285741806,
|
|
"sampling/importance_sampling_ratio/min": 0.7316789954900742,
|
|
"sampling/sampling_logp_difference/max": 0.34203912019729615,
|
|
"sampling/sampling_logp_difference/mean": 0.021399160940200092,
|
|
"step": 8130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 322.9,
|
|
"completions/max_terminated_length": 318.5,
|
|
"completions/mean_length": 235.1125,
|
|
"completions/mean_terminated_length": 231.27916870117187,
|
|
"completions/min_length": 163.6,
|
|
"completions/min_terminated_length": 163.6,
|
|
"entropy": 1.1023086324334144,
|
|
"epoch": 0.7483681162085134,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.571066856384277,
|
|
"kl": 0.05457422552863136,
|
|
"learning_rate": 4.475820704411102e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 35384046.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.38550153374671936,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.3855015397071838,
|
|
"sampling/importance_sampling_ratio/max": 1.3074329614639282,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999190926551819,
|
|
"sampling/importance_sampling_ratio/min": 0.7737188220024109,
|
|
"sampling/sampling_logp_difference/max": 0.2918028891086578,
|
|
"sampling/sampling_logp_difference/mean": 0.022051671892404555,
|
|
"step": 8140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.8,
|
|
"completions/max_terminated_length": 310.8,
|
|
"completions/mean_length": 210.05,
|
|
"completions/mean_terminated_length": 210.05,
|
|
"completions/min_length": 142.6,
|
|
"completions/min_terminated_length": 142.6,
|
|
"entropy": 0.9659396767616272,
|
|
"epoch": 0.7492874873586467,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.13347794115543365,
|
|
"kl": 0.04572153322515078,
|
|
"learning_rate": 4.474267379994895e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 35425834.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.17422050833702088,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.17422052025794982,
|
|
"sampling/importance_sampling_ratio/max": 1.2609246611595153,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993985891342163,
|
|
"sampling/importance_sampling_ratio/min": 0.7660029411315918,
|
|
"sampling/sampling_logp_difference/max": 0.28041499853134155,
|
|
"sampling/sampling_logp_difference/mean": 0.01976697351783514,
|
|
"step": 8150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.4,
|
|
"completions/max_terminated_length": 287.4,
|
|
"completions/mean_length": 197.0875,
|
|
"completions/mean_terminated_length": 197.0875,
|
|
"completions/min_length": 125.7,
|
|
"completions/min_terminated_length": 125.7,
|
|
"entropy": 1.0424822442233563,
|
|
"epoch": 0.75020685850878,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1294274628162384,
|
|
"kl": 0.053996150474995376,
|
|
"learning_rate": 4.4727120278478335e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 35472529.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.3302692323923111,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.3302692502737045,
|
|
"sampling/importance_sampling_ratio/max": 1.25041024684906,
|
|
"sampling/importance_sampling_ratio/mean": 1.000388777256012,
|
|
"sampling/importance_sampling_ratio/min": 0.7827749192714691,
|
|
"sampling/sampling_logp_difference/max": 0.2536800980567932,
|
|
"sampling/sampling_logp_difference/mean": 0.020770432986319066,
|
|
"step": 8160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.0,
|
|
"completions/max_terminated_length": 285.0,
|
|
"completions/mean_length": 218.45,
|
|
"completions/mean_terminated_length": 218.45,
|
|
"completions/min_length": 155.9,
|
|
"completions/min_terminated_length": 155.9,
|
|
"entropy": 1.1214228980243206,
|
|
"epoch": 0.7511262296589133,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.1323217749595642,
|
|
"kl": 0.04653022048296407,
|
|
"learning_rate": 4.471154649567386e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 35518901.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.15235702097415924,
|
|
"sampling/importance_sampling_ratio/max": 1.2770470976829529,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999829590320587,
|
|
"sampling/importance_sampling_ratio/min": 0.7996057629585266,
|
|
"sampling/sampling_logp_difference/max": 0.24931695461273193,
|
|
"sampling/sampling_logp_difference/mean": 0.02149878665804863,
|
|
"step": 8170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.2,
|
|
"completions/max_terminated_length": 278.2,
|
|
"completions/mean_length": 207.8625,
|
|
"completions/mean_terminated_length": 207.8625,
|
|
"completions/min_length": 138.9,
|
|
"completions/min_terminated_length": 138.9,
|
|
"entropy": 1.0842328920960427,
|
|
"epoch": 0.7520456008090466,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.630114555358887,
|
|
"kl": 0.045263630733825264,
|
|
"learning_rate": 4.469595246753107e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 35563930.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.3311904340982437,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.3311904460191727,
|
|
"sampling/importance_sampling_ratio/max": 1.2654990434646607,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001943409442902,
|
|
"sampling/importance_sampling_ratio/min": 0.79794762134552,
|
|
"sampling/sampling_logp_difference/max": 0.2562193155288696,
|
|
"sampling/sampling_logp_difference/mean": 0.021051904559135436,
|
|
"step": 8180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.5,
|
|
"completions/max_terminated_length": 307.5,
|
|
"completions/mean_length": 227.0125,
|
|
"completions/mean_terminated_length": 227.0125,
|
|
"completions/min_length": 135.6,
|
|
"completions/min_terminated_length": 135.6,
|
|
"entropy": 1.1410873807966708,
|
|
"epoch": 0.7529649719591799,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.307127475738525,
|
|
"kl": 0.0383632734650746,
|
|
"learning_rate": 4.4680338210066255e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 35607331.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.26819893419742585,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.2681989401578903,
|
|
"sampling/importance_sampling_ratio/max": 1.2538456916809082,
|
|
"sampling/importance_sampling_ratio/mean": 1.000129634141922,
|
|
"sampling/importance_sampling_ratio/min": 0.7794063210487365,
|
|
"sampling/sampling_logp_difference/max": 0.2568260908126831,
|
|
"sampling/sampling_logp_difference/mean": 0.021609990578144788,
|
|
"step": 8190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.2,
|
|
"completions/max_terminated_length": 292.2,
|
|
"completions/mean_length": 221.9,
|
|
"completions/mean_terminated_length": 221.9,
|
|
"completions/min_length": 161.2,
|
|
"completions/min_terminated_length": 161.2,
|
|
"entropy": 1.1074596807360648,
|
|
"epoch": 0.7538843431093132,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14383909106254578,
|
|
"kl": 0.03385354392230511,
|
|
"learning_rate": 4.466470373931654e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 35651603.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.39694315791130064,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.39694316387176515,
|
|
"sampling/importance_sampling_ratio/max": 1.2885071754455566,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000363349914552,
|
|
"sampling/importance_sampling_ratio/min": 0.740203732252121,
|
|
"sampling/sampling_logp_difference/max": 0.31600978374481203,
|
|
"sampling/sampling_logp_difference/mean": 0.021144175529479982,
|
|
"step": 8200
|
|
},
|
|
{
|
|
"epoch": 0.7538843431093132,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0008726003490401396,
|
|
"eval_completions/max_length": 293.4066317626527,
|
|
"eval_completions/max_terminated_length": 292.9738219895288,
|
|
"eval_completions/mean_length": 206.5868237347295,
|
|
"eval_completions/mean_terminated_length": 206.39631644843138,
|
|
"eval_completions/min_length": 136.28097731239092,
|
|
"eval_completions/min_terminated_length": 136.28097731239092,
|
|
"eval_entropy": 1.0882980898932846,
|
|
"eval_frac_reward_zero_std": 0.3263525305410122,
|
|
"eval_kl": 0.041748599560498985,
|
|
"eval_loss": 0.0021079054567962885,
|
|
"eval_num_tokens": 35651603.0,
|
|
"eval_reward": 0.060536649214659684,
|
|
"eval_reward_std": 0.34079768567185126,
|
|
"eval_rewards/reward_fn/mean": 0.060536649214659684,
|
|
"eval_rewards/reward_fn/std": 0.34079769690623457,
|
|
"eval_runtime": 1714.2684,
|
|
"eval_samples_per_second": 0.334,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2691617898291943,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999526114988078,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.775991971086041,
|
|
"eval_sampling/sampling_logp_difference/max": 0.28232999689499955,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020957368091203882,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 8200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 335.0,
|
|
"completions/max_terminated_length": 319.3,
|
|
"completions/mean_length": 223.9375,
|
|
"completions/mean_terminated_length": 221.0017883300781,
|
|
"completions/min_length": 139.1,
|
|
"completions/min_terminated_length": 139.1,
|
|
"entropy": 1.1713350355625152,
|
|
"epoch": 0.7548037142594466,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.09596233814954758,
|
|
"kl": 0.0412192597577814,
|
|
"learning_rate": 4.464904907133976e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 35698430.0,
|
|
"reward": -0.03125,
|
|
"reward_std": 0.30431793332099916,
|
|
"rewards/reward_fn/mean": -0.03125,
|
|
"rewards/reward_fn/std": 0.3043179452419281,
|
|
"sampling/importance_sampling_ratio/max": 1.2394259333610536,
|
|
"sampling/importance_sampling_ratio/mean": 0.9992289543151855,
|
|
"sampling/importance_sampling_ratio/min": 0.753474485874176,
|
|
"sampling/sampling_logp_difference/max": 0.28635513484478,
|
|
"sampling/sampling_logp_difference/mean": 0.02260472234338522,
|
|
"step": 8210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.5,
|
|
"completions/max_terminated_length": 289.5,
|
|
"completions/mean_length": 211.5125,
|
|
"completions/mean_terminated_length": 211.5125,
|
|
"completions/min_length": 149.9,
|
|
"completions/min_terminated_length": 149.9,
|
|
"entropy": 1.1306281983852386,
|
|
"epoch": 0.7557230854095799,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.317781925201416,
|
|
"kl": 0.04723062192206271,
|
|
"learning_rate": 4.463337422221452e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 35741655.0,
|
|
"reward": -0.3375,
|
|
"reward_std": 0.4089031368494034,
|
|
"rewards/reward_fn/mean": -0.3375,
|
|
"rewards/reward_fn/std": 0.40890314280986784,
|
|
"sampling/importance_sampling_ratio/max": 1.2793290138244628,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998090744018555,
|
|
"sampling/importance_sampling_ratio/min": 0.7788514912128448,
|
|
"sampling/sampling_logp_difference/max": 0.3048470139503479,
|
|
"sampling/sampling_logp_difference/mean": 0.02157476041465998,
|
|
"step": 8220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.0,
|
|
"completions/max_terminated_length": 292.0,
|
|
"completions/mean_length": 211.8125,
|
|
"completions/mean_terminated_length": 211.8125,
|
|
"completions/min_length": 139.6,
|
|
"completions/min_terminated_length": 139.6,
|
|
"entropy": 1.1112032204866409,
|
|
"epoch": 0.7566424565597132,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.12457574903964996,
|
|
"kl": 0.04086814378388226,
|
|
"learning_rate": 4.461767920804015e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 35784832.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.24485630989074708,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.24485631585121154,
|
|
"sampling/importance_sampling_ratio/max": 1.2734048962593079,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002968907356262,
|
|
"sampling/importance_sampling_ratio/min": 0.7815337657928467,
|
|
"sampling/sampling_logp_difference/max": 0.2746634900569916,
|
|
"sampling/sampling_logp_difference/mean": 0.02131683360785246,
|
|
"step": 8230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.8,
|
|
"completions/max_terminated_length": 268.8,
|
|
"completions/mean_length": 191.975,
|
|
"completions/mean_terminated_length": 191.975,
|
|
"completions/min_length": 129.6,
|
|
"completions/min_terminated_length": 129.6,
|
|
"entropy": 1.027645231038332,
|
|
"epoch": 0.7575618277098465,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.389520168304443,
|
|
"kl": 0.06212011110037565,
|
|
"learning_rate": 4.460196404493669e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 35829398.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.33751319646835326,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.3375132083892822,
|
|
"sampling/importance_sampling_ratio/max": 1.2784918308258058,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001433610916137,
|
|
"sampling/importance_sampling_ratio/min": 0.7732639849185944,
|
|
"sampling/sampling_logp_difference/max": 0.2856587052345276,
|
|
"sampling/sampling_logp_difference/mean": 0.020899677462875844,
|
|
"step": 8240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.9,
|
|
"completions/max_terminated_length": 294.9,
|
|
"completions/mean_length": 206.475,
|
|
"completions/mean_terminated_length": 206.475,
|
|
"completions/min_length": 128.2,
|
|
"completions/min_terminated_length": 128.2,
|
|
"entropy": 1.0497943833470345,
|
|
"epoch": 0.7584811988599798,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.15488189458847046,
|
|
"kl": 0.05923373339464888,
|
|
"learning_rate": 4.458622874904486e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 35871420.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.15235702097415924,
|
|
"sampling/importance_sampling_ratio/max": 1.2728855848312377,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003418028354645,
|
|
"sampling/importance_sampling_ratio/min": 0.7701029598712921,
|
|
"sampling/sampling_logp_difference/max": 0.2737884342670441,
|
|
"sampling/sampling_logp_difference/mean": 0.02055224124342203,
|
|
"step": 8250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.9,
|
|
"completions/max_terminated_length": 281.9,
|
|
"completions/mean_length": 197.1125,
|
|
"completions/mean_terminated_length": 197.1125,
|
|
"completions/min_length": 117.3,
|
|
"completions/min_terminated_length": 117.3,
|
|
"entropy": 1.1788907170295715,
|
|
"epoch": 0.7594005700101131,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.557623863220215,
|
|
"kl": 0.050654699525330214,
|
|
"learning_rate": 4.457047333652607e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 35916149.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3554430425167084,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.3554430603981018,
|
|
"sampling/importance_sampling_ratio/max": 1.2642478227615357,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001820981502534,
|
|
"sampling/importance_sampling_ratio/min": 0.7880287289619445,
|
|
"sampling/sampling_logp_difference/max": 0.2631549179553986,
|
|
"sampling/sampling_logp_difference/mean": 0.023391783982515336,
|
|
"step": 8260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.3,
|
|
"completions/max_terminated_length": 255.3,
|
|
"completions/mean_length": 178.975,
|
|
"completions/mean_terminated_length": 178.975,
|
|
"completions/min_length": 112.8,
|
|
"completions/min_terminated_length": 112.8,
|
|
"entropy": 0.984667555987835,
|
|
"epoch": 0.7603199411602464,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.660867691040039,
|
|
"kl": 0.05467489522416145,
|
|
"learning_rate": 4.455469782356241e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 35955395.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.37809050679206846,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.37809052467346194,
|
|
"sampling/importance_sampling_ratio/max": 1.2664636731147767,
|
|
"sampling/importance_sampling_ratio/mean": 1.0007215559482574,
|
|
"sampling/importance_sampling_ratio/min": 0.792838740348816,
|
|
"sampling/sampling_logp_difference/max": 0.2565943658351898,
|
|
"sampling/sampling_logp_difference/mean": 0.020006229542195796,
|
|
"step": 8270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 278.1,
|
|
"completions/max_terminated_length": 258.0,
|
|
"completions/mean_length": 185.1,
|
|
"completions/mean_terminated_length": 181.7964294433594,
|
|
"completions/min_length": 115.7,
|
|
"completions/min_terminated_length": 115.7,
|
|
"entropy": 1.017738151550293,
|
|
"epoch": 0.7612393123103797,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.6502814292907715,
|
|
"kl": 0.05798605147283524,
|
|
"learning_rate": 4.453890222635657e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 35992907.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.4846843808889389,
|
|
"rewards/reward_fn/mean": 0.09375,
|
|
"rewards/reward_fn/std": 0.4846844047307968,
|
|
"sampling/importance_sampling_ratio/max": 1.3162721157073975,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002390563488006,
|
|
"sampling/importance_sampling_ratio/min": 0.7624096393585205,
|
|
"sampling/sampling_logp_difference/max": 0.3455509454011917,
|
|
"sampling/sampling_logp_difference/mean": 0.020331501960754395,
|
|
"step": 8280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.6,
|
|
"completions/max_terminated_length": 284.6,
|
|
"completions/mean_length": 205.85,
|
|
"completions/mean_terminated_length": 205.85,
|
|
"completions/min_length": 127.4,
|
|
"completions/min_terminated_length": 127.4,
|
|
"entropy": 1.132000409066677,
|
|
"epoch": 0.762158683460513,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.36728048324585,
|
|
"kl": 0.05303168012178503,
|
|
"learning_rate": 4.452308656113191e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 36037607.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.1948736011981964,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.1948736011981964,
|
|
"sampling/importance_sampling_ratio/max": 1.2345200657844544,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002072334289551,
|
|
"sampling/importance_sampling_ratio/min": 0.7656309545040131,
|
|
"sampling/sampling_logp_difference/max": 0.27644087076187135,
|
|
"sampling/sampling_logp_difference/mean": 0.021915152855217456,
|
|
"step": 8290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.7,
|
|
"completions/max_terminated_length": 281.7,
|
|
"completions/mean_length": 195.025,
|
|
"completions/mean_terminated_length": 195.025,
|
|
"completions/min_length": 127.0,
|
|
"completions/min_terminated_length": 127.0,
|
|
"entropy": 1.1078555762767792,
|
|
"epoch": 0.7630780546106464,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.462594509124756,
|
|
"kl": 0.05632182351546362,
|
|
"learning_rate": 4.45072508441324e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 36079249.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.2903633385896683,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.2903633564710617,
|
|
"sampling/importance_sampling_ratio/max": 1.2656308650970458,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998750507831573,
|
|
"sampling/importance_sampling_ratio/min": 0.777303683757782,
|
|
"sampling/sampling_logp_difference/max": 0.28412245512008666,
|
|
"sampling/sampling_logp_difference/mean": 0.02181570455431938,
|
|
"step": 8300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.3,
|
|
"completions/max_terminated_length": 298.3,
|
|
"completions/mean_length": 203.3,
|
|
"completions/mean_terminated_length": 203.3,
|
|
"completions/min_length": 129.9,
|
|
"completions/min_terminated_length": 129.9,
|
|
"entropy": 1.2150594726204873,
|
|
"epoch": 0.7639974257607797,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.037957668304443,
|
|
"kl": 0.062250671011861415,
|
|
"learning_rate": 4.449139509162257e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 36122201.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.2584230363368988,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.2584230363368988,
|
|
"sampling/importance_sampling_ratio/max": 1.2610936641693116,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001720666885376,
|
|
"sampling/importance_sampling_ratio/min": 0.7996842503547669,
|
|
"sampling/sampling_logp_difference/max": 0.25071250200271605,
|
|
"sampling/sampling_logp_difference/mean": 0.022956776432693005,
|
|
"step": 8310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.7,
|
|
"completions/max_terminated_length": 268.7,
|
|
"completions/mean_length": 203.0375,
|
|
"completions/mean_terminated_length": 203.0375,
|
|
"completions/min_length": 135.8,
|
|
"completions/min_terminated_length": 135.8,
|
|
"entropy": 1.085725224763155,
|
|
"epoch": 0.764916796910913,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1267528533935547,
|
|
"kl": 0.04891264250036329,
|
|
"learning_rate": 4.447551931988755e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 36162444.0,
|
|
"reward": 0.01875,
|
|
"reward_std": 0.22178960591554642,
|
|
"rewards/reward_fn/mean": 0.01875,
|
|
"rewards/reward_fn/std": 0.2217896118760109,
|
|
"sampling/importance_sampling_ratio/max": 1.3326180219650268,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002044022083283,
|
|
"sampling/importance_sampling_ratio/min": 0.7759069800376892,
|
|
"sampling/sampling_logp_difference/max": 0.3022372007369995,
|
|
"sampling/sampling_logp_difference/mean": 0.021860329993069173,
|
|
"step": 8320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 335.2,
|
|
"completions/max_terminated_length": 335.2,
|
|
"completions/mean_length": 224.9375,
|
|
"completions/mean_terminated_length": 224.9375,
|
|
"completions/min_length": 145.6,
|
|
"completions/min_terminated_length": 145.6,
|
|
"entropy": 1.1673907034099102,
|
|
"epoch": 0.7658361680610463,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.223412036895752,
|
|
"kl": 0.06102771374862641,
|
|
"learning_rate": 4.445962354523304e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 36207863.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.3955783486366272,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.39557836055755613,
|
|
"sampling/importance_sampling_ratio/max": 1.2773509860038756,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999799132347107,
|
|
"sampling/importance_sampling_ratio/min": 0.7693881154060364,
|
|
"sampling/sampling_logp_difference/max": 0.2882683277130127,
|
|
"sampling/sampling_logp_difference/mean": 0.0226906631141901,
|
|
"step": 8330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 321.3,
|
|
"completions/max_terminated_length": 321.3,
|
|
"completions/mean_length": 222.9125,
|
|
"completions/mean_terminated_length": 222.9125,
|
|
"completions/min_length": 139.2,
|
|
"completions/min_terminated_length": 139.2,
|
|
"entropy": 1.0983840510249139,
|
|
"epoch": 0.7667555392111796,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.316318511962891,
|
|
"kl": 0.03722487888298929,
|
|
"learning_rate": 4.4443707783985255e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 36250808.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.2592615157365799,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.2592615157365799,
|
|
"sampling/importance_sampling_ratio/max": 1.2922040939331054,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997156381607055,
|
|
"sampling/importance_sampling_ratio/min": 0.7839018762111664,
|
|
"sampling/sampling_logp_difference/max": 0.2833583474159241,
|
|
"sampling/sampling_logp_difference/mean": 0.021531708538532257,
|
|
"step": 8340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.6,
|
|
"completions/max_terminated_length": 287.6,
|
|
"completions/mean_length": 215.275,
|
|
"completions/mean_terminated_length": 215.275,
|
|
"completions/min_length": 149.2,
|
|
"completions/min_terminated_length": 149.2,
|
|
"entropy": 1.0162071652710438,
|
|
"epoch": 0.7676749103613129,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12623989582061768,
|
|
"kl": 0.03177037722198293,
|
|
"learning_rate": 4.442777205249098e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 36294742.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.2958350956439972,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.29583510756492615,
|
|
"sampling/importance_sampling_ratio/max": 1.249613606929779,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001385152339934,
|
|
"sampling/importance_sampling_ratio/min": 0.7501398980617523,
|
|
"sampling/sampling_logp_difference/max": 0.2947612404823303,
|
|
"sampling/sampling_logp_difference/mean": 0.01980518698692322,
|
|
"step": 8350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 339.9,
|
|
"completions/max_terminated_length": 339.9,
|
|
"completions/mean_length": 216.625,
|
|
"completions/mean_terminated_length": 216.625,
|
|
"completions/min_length": 127.4,
|
|
"completions/min_terminated_length": 127.4,
|
|
"entropy": 1.0463447362184524,
|
|
"epoch": 0.7685942815114462,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.614830493927002,
|
|
"kl": 0.050177161960164085,
|
|
"learning_rate": 4.441181636711749e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 36338264.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3055525213479996,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.3055525213479996,
|
|
"sampling/importance_sampling_ratio/max": 1.316318929195404,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998241186141967,
|
|
"sampling/importance_sampling_ratio/min": 0.784780102968216,
|
|
"sampling/sampling_logp_difference/max": 0.3033287763595581,
|
|
"sampling/sampling_logp_difference/mean": 0.020822701789438725,
|
|
"step": 8360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 326.7,
|
|
"completions/max_terminated_length": 316.6,
|
|
"completions/mean_length": 219.575,
|
|
"completions/mean_terminated_length": 216.75,
|
|
"completions/min_length": 112.9,
|
|
"completions/min_terminated_length": 112.9,
|
|
"entropy": 1.100361604243517,
|
|
"epoch": 0.7695136526615795,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.0107035636901855,
|
|
"kl": 0.05771958347177133,
|
|
"learning_rate": 4.439584074425253e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 36382454.0,
|
|
"reward": -0.04375,
|
|
"reward_std": 0.38596427738666533,
|
|
"rewards/reward_fn/mean": -0.04375,
|
|
"rewards/reward_fn/std": 0.3859642893075943,
|
|
"sampling/importance_sampling_ratio/max": 1.3737538933753968,
|
|
"sampling/importance_sampling_ratio/mean": 1.000322026014328,
|
|
"sampling/importance_sampling_ratio/min": 0.7813075542449951,
|
|
"sampling/sampling_logp_difference/max": 0.32148070335388185,
|
|
"sampling/sampling_logp_difference/mean": 0.02222872506827116,
|
|
"step": 8370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 342.7,
|
|
"completions/max_terminated_length": 330.1,
|
|
"completions/mean_length": 248.0125,
|
|
"completions/mean_terminated_length": 246.0232147216797,
|
|
"completions/min_length": 180.3,
|
|
"completions/min_terminated_length": 180.3,
|
|
"entropy": 1.088571298867464,
|
|
"epoch": 0.7704330238117127,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.949433326721191,
|
|
"kl": 0.057909214548999444,
|
|
"learning_rate": 4.437984520030437e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 36427879.0,
|
|
"reward": -0.00625,
|
|
"reward_std": 0.3686827152967453,
|
|
"rewards/reward_fn/mean": -0.00625,
|
|
"rewards/reward_fn/std": 0.3686827152967453,
|
|
"sampling/importance_sampling_ratio/max": 1.2526766061782837,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005345821380616,
|
|
"sampling/importance_sampling_ratio/min": 0.7728150308132171,
|
|
"sampling/sampling_logp_difference/max": 0.2652811765670776,
|
|
"sampling/sampling_logp_difference/mean": 0.021244097501039505,
|
|
"step": 8380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.7,
|
|
"completions/max_terminated_length": 319.7,
|
|
"completions/mean_length": 214.3125,
|
|
"completions/mean_terminated_length": 214.3125,
|
|
"completions/min_length": 141.9,
|
|
"completions/min_terminated_length": 141.9,
|
|
"entropy": 1.1342267915606499,
|
|
"epoch": 0.771352394961846,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.30804443359375,
|
|
"kl": 0.049994938942836596,
|
|
"learning_rate": 4.43638297517017e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 36472184.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.2797048300504684,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.27970483601093293,
|
|
"sampling/importance_sampling_ratio/max": 1.3075689911842345,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997446179389954,
|
|
"sampling/importance_sampling_ratio/min": 0.7942805528640747,
|
|
"sampling/sampling_logp_difference/max": 0.2780471324920654,
|
|
"sampling/sampling_logp_difference/mean": 0.022112540528178216,
|
|
"step": 8390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 341.8,
|
|
"completions/max_terminated_length": 341.8,
|
|
"completions/mean_length": 240.4875,
|
|
"completions/mean_terminated_length": 240.4875,
|
|
"completions/min_length": 154.8,
|
|
"completions/min_terminated_length": 154.8,
|
|
"entropy": 1.1278797470033168,
|
|
"epoch": 0.7722717661119795,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.3170485496521,
|
|
"kl": 0.04881764885503799,
|
|
"learning_rate": 4.434779441489367e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 36519879.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.28029437065124513,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.28029437065124513,
|
|
"sampling/importance_sampling_ratio/max": 1.3026459097862244,
|
|
"sampling/importance_sampling_ratio/mean": 1.000385767221451,
|
|
"sampling/importance_sampling_ratio/min": 0.7353048473596573,
|
|
"sampling/sampling_logp_difference/max": 0.35137550830841063,
|
|
"sampling/sampling_logp_difference/mean": 0.02200442273169756,
|
|
"step": 8400
|
|
},
|
|
{
|
|
"epoch": 0.7722717661119795,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.005671902268760907,
|
|
"eval_completions/max_length": 330.49912739965094,
|
|
"eval_completions/max_terminated_length": 326.1064572425829,
|
|
"eval_completions/mean_length": 234.5556282722513,
|
|
"eval_completions/mean_terminated_length": 233.35154417880128,
|
|
"eval_completions/min_length": 158.89877835951134,
|
|
"eval_completions/min_terminated_length": 158.89877835951134,
|
|
"eval_entropy": 1.11524294308967,
|
|
"eval_frac_reward_zero_std": 0.31413612565445026,
|
|
"eval_kl": 0.04713486554832515,
|
|
"eval_loss": 0.0024906403850764036,
|
|
"eval_num_tokens": 36519879.0,
|
|
"eval_reward": 0.053773996509598604,
|
|
"eval_reward_std": 0.3388520177018164,
|
|
"eval_rewards/reward_fn/mean": 0.053773996509598604,
|
|
"eval_rewards/reward_fn/std": 0.33885202560749356,
|
|
"eval_runtime": 1904.3541,
|
|
"eval_samples_per_second": 0.301,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.276251434863758,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999885669345423,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7712784995167251,
|
|
"eval_sampling/sampling_logp_difference/max": 0.28693026538294647,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021595409305775977,
|
|
"eval_steps_per_second": 0.038,
|
|
"step": 8400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 342.1,
|
|
"completions/max_terminated_length": 342.1,
|
|
"completions/mean_length": 231.85,
|
|
"completions/mean_terminated_length": 231.85,
|
|
"completions/min_length": 148.5,
|
|
"completions/min_terminated_length": 148.5,
|
|
"entropy": 1.112854889780283,
|
|
"epoch": 0.7731911372621127,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.385634899139404,
|
|
"kl": 0.05270067517412826,
|
|
"learning_rate": 4.433173920634986e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 36566155.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.2972315102815628,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.29723151624202726,
|
|
"sampling/importance_sampling_ratio/max": 1.294772219657898,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998943567276001,
|
|
"sampling/importance_sampling_ratio/min": 0.7979962587356567,
|
|
"sampling/sampling_logp_difference/max": 0.2656439423561096,
|
|
"sampling/sampling_logp_difference/mean": 0.021819342859089375,
|
|
"step": 8410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 323.2,
|
|
"completions/max_terminated_length": 323.2,
|
|
"completions/mean_length": 223.35,
|
|
"completions/mean_terminated_length": 223.35,
|
|
"completions/min_length": 149.9,
|
|
"completions/min_terminated_length": 149.9,
|
|
"entropy": 1.0673673093318938,
|
|
"epoch": 0.774110508412246,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.54659366607666,
|
|
"kl": 0.03772438338492066,
|
|
"learning_rate": 4.431566414256026e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 36611103.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.2687691628932953,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.2687691688537598,
|
|
"sampling/importance_sampling_ratio/max": 1.267129933834076,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002500474452973,
|
|
"sampling/importance_sampling_ratio/min": 0.7844542920589447,
|
|
"sampling/sampling_logp_difference/max": 0.25771968364715575,
|
|
"sampling/sampling_logp_difference/mean": 0.0217557854950428,
|
|
"step": 8420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.1,
|
|
"completions/max_terminated_length": 307.1,
|
|
"completions/mean_length": 221.9875,
|
|
"completions/mean_terminated_length": 221.9875,
|
|
"completions/min_length": 149.3,
|
|
"completions/min_terminated_length": 149.3,
|
|
"entropy": 1.18791157156229,
|
|
"epoch": 0.7750298795623793,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.5652642250061035,
|
|
"kl": 0.05099186827428639,
|
|
"learning_rate": 4.4299569240035245e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 36659958.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.3742649495601654,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.3742649555206299,
|
|
"sampling/importance_sampling_ratio/max": 1.2980244398117065,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004709303379058,
|
|
"sampling/importance_sampling_ratio/min": 0.7510793507099152,
|
|
"sampling/sampling_logp_difference/max": 0.31801933646202085,
|
|
"sampling/sampling_logp_difference/mean": 0.023117323778569697,
|
|
"step": 8430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 306.6,
|
|
"completions/max_terminated_length": 287.7,
|
|
"completions/mean_length": 219.3625,
|
|
"completions/mean_terminated_length": 216.1732147216797,
|
|
"completions/min_length": 150.2,
|
|
"completions/min_terminated_length": 150.2,
|
|
"entropy": 1.1362251348793506,
|
|
"epoch": 0.7759492507125126,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.763079643249512,
|
|
"kl": 0.0665880466811359,
|
|
"learning_rate": 4.428345451530557e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 36704003.0,
|
|
"reward": -0.16875,
|
|
"reward_std": 0.2571449428796768,
|
|
"rewards/reward_fn/mean": -0.16875,
|
|
"rewards/reward_fn/std": 0.2571449488401413,
|
|
"sampling/importance_sampling_ratio/max": 1.2639767050743103,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002585172653198,
|
|
"sampling/importance_sampling_ratio/min": 0.7485855638980865,
|
|
"sampling/sampling_logp_difference/max": 0.29659217596054077,
|
|
"sampling/sampling_logp_difference/mean": 0.022258792631328107,
|
|
"step": 8440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.2,
|
|
"completions/max_terminated_length": 272.2,
|
|
"completions/mean_length": 192.9625,
|
|
"completions/mean_terminated_length": 192.9625,
|
|
"completions/min_length": 132.7,
|
|
"completions/min_terminated_length": 132.7,
|
|
"entropy": 0.9923920854926109,
|
|
"epoch": 0.7768686218626459,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.36787649989128113,
|
|
"kl": 0.05050960383960046,
|
|
"learning_rate": 4.426731998492236e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 36741600.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.12793734967708587,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.12793734967708587,
|
|
"sampling/importance_sampling_ratio/max": 1.259957730770111,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999345541000366,
|
|
"sampling/importance_sampling_ratio/min": 0.7655860543251037,
|
|
"sampling/sampling_logp_difference/max": 0.29106817245483396,
|
|
"sampling/sampling_logp_difference/mean": 0.021167710050940515,
|
|
"step": 8450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 290.1,
|
|
"completions/max_terminated_length": 279.8,
|
|
"completions/mean_length": 202.25,
|
|
"completions/mean_terminated_length": 199.81785888671874,
|
|
"completions/min_length": 140.8,
|
|
"completions/min_terminated_length": 140.8,
|
|
"entropy": 1.0233232654631137,
|
|
"epoch": 0.7777879930127792,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.527560710906982,
|
|
"kl": 0.048779568687314166,
|
|
"learning_rate": 4.425116566545706e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 36786372.0,
|
|
"reward": -0.06875,
|
|
"reward_std": 0.45328661799430847,
|
|
"rewards/reward_fn/mean": -0.06875,
|
|
"rewards/reward_fn/std": 0.45328662991523744,
|
|
"sampling/importance_sampling_ratio/max": 1.330122721195221,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000096380710601,
|
|
"sampling/importance_sampling_ratio/min": 0.7576094567775726,
|
|
"sampling/sampling_logp_difference/max": 0.3212560206651688,
|
|
"sampling/sampling_logp_difference/mean": 0.021010516956448556,
|
|
"step": 8460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 315.5,
|
|
"completions/max_terminated_length": 303.7,
|
|
"completions/mean_length": 222.3,
|
|
"completions/mean_terminated_length": 219.8,
|
|
"completions/min_length": 142.3,
|
|
"completions/min_terminated_length": 142.3,
|
|
"entropy": 1.1069677479565143,
|
|
"epoch": 0.7787073641629125,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.056061267852783,
|
|
"kl": 0.05582822843571193,
|
|
"learning_rate": 4.4234991573501426e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 36832700.0,
|
|
"reward": 0.01875,
|
|
"reward_std": 0.40446448028087617,
|
|
"rewards/reward_fn/mean": 0.01875,
|
|
"rewards/reward_fn/std": 0.4044644981622696,
|
|
"sampling/importance_sampling_ratio/max": 1.2818525791168214,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005426943302154,
|
|
"sampling/importance_sampling_ratio/min": 0.7787148594856262,
|
|
"sampling/sampling_logp_difference/max": 0.2689772129058838,
|
|
"sampling/sampling_logp_difference/mean": 0.021798519976437093,
|
|
"step": 8470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.8,
|
|
"completions/max_terminated_length": 309.8,
|
|
"completions/mean_length": 213.5,
|
|
"completions/mean_terminated_length": 213.5,
|
|
"completions/min_length": 123.8,
|
|
"completions/min_terminated_length": 123.8,
|
|
"entropy": 1.1976394198834897,
|
|
"epoch": 0.7796267353130458,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.758187294006348,
|
|
"kl": 0.04055676264688372,
|
|
"learning_rate": 4.42187977256676e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 36876428.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.3095962584018707,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.3095962584018707,
|
|
"sampling/importance_sampling_ratio/max": 1.3068729877471923,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999541759490966,
|
|
"sampling/importance_sampling_ratio/min": 0.7789978623390198,
|
|
"sampling/sampling_logp_difference/max": 0.303797447681427,
|
|
"sampling/sampling_logp_difference/mean": 0.022847374342381953,
|
|
"step": 8480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.2,
|
|
"completions/max_terminated_length": 315.2,
|
|
"completions/mean_length": 204.15,
|
|
"completions/mean_terminated_length": 204.15,
|
|
"completions/min_length": 144.5,
|
|
"completions/min_terminated_length": 144.5,
|
|
"entropy": 1.0322167932987214,
|
|
"epoch": 0.7805461064631792,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.10412130504846573,
|
|
"kl": 0.040058744698762895,
|
|
"learning_rate": 4.420258413858792e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 36916408.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.2205115258693695,
|
|
"sampling/importance_sampling_ratio/max": 1.312811589241028,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000422835350036,
|
|
"sampling/importance_sampling_ratio/min": 0.7942772030830383,
|
|
"sampling/sampling_logp_difference/max": 0.277325439453125,
|
|
"sampling/sampling_logp_difference/mean": 0.02104961033910513,
|
|
"step": 8490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.6,
|
|
"completions/max_terminated_length": 307.6,
|
|
"completions/mean_length": 221.175,
|
|
"completions/mean_terminated_length": 221.175,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.1144550062716008,
|
|
"epoch": 0.7814654776133125,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13495837152004242,
|
|
"kl": 0.04637873565661721,
|
|
"learning_rate": 4.418635082891506e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 36962110.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.2765199452638626,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.2765199452638626,
|
|
"sampling/importance_sampling_ratio/max": 1.2671054244041442,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998886525630951,
|
|
"sampling/importance_sampling_ratio/min": 0.7846600711345673,
|
|
"sampling/sampling_logp_difference/max": 0.2706998348236084,
|
|
"sampling/sampling_logp_difference/mean": 0.021386586502194403,
|
|
"step": 8500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 317.0,
|
|
"completions/max_terminated_length": 317.0,
|
|
"completions/mean_length": 213.8,
|
|
"completions/mean_terminated_length": 213.8,
|
|
"completions/min_length": 142.2,
|
|
"completions/min_terminated_length": 142.2,
|
|
"entropy": 1.0674708582460881,
|
|
"epoch": 0.7823848487634458,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.907252311706543,
|
|
"kl": 0.04561225168872625,
|
|
"learning_rate": 4.417009781332192e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 37007134.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.35502841174602506,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.3550284177064896,
|
|
"sampling/importance_sampling_ratio/max": 1.2598554491996765,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996153533458709,
|
|
"sampling/importance_sampling_ratio/min": 0.7693885743618012,
|
|
"sampling/sampling_logp_difference/max": 0.2738891065120697,
|
|
"sampling/sampling_logp_difference/mean": 0.021134657226502894,
|
|
"step": 8510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.0,
|
|
"completions/max_terminated_length": 297.0,
|
|
"completions/mean_length": 213.0375,
|
|
"completions/mean_terminated_length": 213.0375,
|
|
"completions/min_length": 134.5,
|
|
"completions/min_terminated_length": 134.5,
|
|
"entropy": 1.042586360499263,
|
|
"epoch": 0.7833042199135791,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.600070953369141,
|
|
"kl": 0.04060280580306426,
|
|
"learning_rate": 4.4153825108501665e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 37047113.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.25210027396678925,
|
|
"rewards/reward_fn/mean": 0.3375,
|
|
"rewards/reward_fn/std": 0.25210027396678925,
|
|
"sampling/importance_sampling_ratio/max": 1.2477716207504272,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993491470813751,
|
|
"sampling/importance_sampling_ratio/min": 0.782102108001709,
|
|
"sampling/sampling_logp_difference/max": 0.258186274766922,
|
|
"sampling/sampling_logp_difference/mean": 0.02058903593569994,
|
|
"step": 8520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 352.2,
|
|
"completions/max_terminated_length": 328.6,
|
|
"completions/mean_length": 227.3625,
|
|
"completions/mean_terminated_length": 220.95178680419923,
|
|
"completions/min_length": 150.3,
|
|
"completions/min_terminated_length": 150.3,
|
|
"entropy": 1.085213840007782,
|
|
"epoch": 0.7842235910637124,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.411600112915039,
|
|
"kl": 0.04907072086352855,
|
|
"learning_rate": 4.4137532731167646e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 37094414.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.3953360438346863,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.3953360617160797,
|
|
"sampling/importance_sampling_ratio/max": 1.3134648680686951,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001985907554627,
|
|
"sampling/importance_sampling_ratio/min": 0.7660737335681915,
|
|
"sampling/sampling_logp_difference/max": 0.32343039512634275,
|
|
"sampling/sampling_logp_difference/mean": 0.021011572517454623,
|
|
"step": 8530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.3,
|
|
"completions/max_terminated_length": 308.3,
|
|
"completions/mean_length": 223.225,
|
|
"completions/mean_terminated_length": 223.225,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 1.0233835220336913,
|
|
"epoch": 0.7851429622138457,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.15121251344680786,
|
|
"kl": 0.061595835560001436,
|
|
"learning_rate": 4.412122069805345e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 37139128.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.31187528371810913,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.31187528371810913,
|
|
"sampling/importance_sampling_ratio/max": 1.2777536392211915,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998096585273742,
|
|
"sampling/importance_sampling_ratio/min": 0.7663146078586578,
|
|
"sampling/sampling_logp_difference/max": 0.2899179458618164,
|
|
"sampling/sampling_logp_difference/mean": 0.02097825426608324,
|
|
"step": 8540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.4,
|
|
"completions/max_terminated_length": 283.4,
|
|
"completions/mean_length": 201.075,
|
|
"completions/mean_terminated_length": 201.075,
|
|
"completions/min_length": 122.8,
|
|
"completions/min_terminated_length": 122.8,
|
|
"entropy": 1.0402743242681027,
|
|
"epoch": 0.786062333363979,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13226866722106934,
|
|
"kl": 0.05446178841521032,
|
|
"learning_rate": 4.410488902591286e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 37182806.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.28210032284259795,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.28210032284259795,
|
|
"sampling/importance_sampling_ratio/max": 1.2301283121109008,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996185660362243,
|
|
"sampling/importance_sampling_ratio/min": 0.7971519410610199,
|
|
"sampling/sampling_logp_difference/max": 0.2421970248222351,
|
|
"sampling/sampling_logp_difference/mean": 0.020698485244065524,
|
|
"step": 8550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.4,
|
|
"completions/max_terminated_length": 279.4,
|
|
"completions/mean_length": 200.75,
|
|
"completions/mean_terminated_length": 200.75,
|
|
"completions/min_length": 127.1,
|
|
"completions/min_terminated_length": 127.1,
|
|
"entropy": 1.077260223031044,
|
|
"epoch": 0.7869817045141123,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.15907880663871765,
|
|
"kl": 0.03778332668007352,
|
|
"learning_rate": 4.408853773151977e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 37224002.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.2912300378084183,
|
|
"rewards/reward_fn/mean": 0.2875,
|
|
"rewards/reward_fn/std": 0.2912300378084183,
|
|
"sampling/importance_sampling_ratio/max": 1.2608644247055054,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998204052448273,
|
|
"sampling/importance_sampling_ratio/min": 0.7914727687835693,
|
|
"sampling/sampling_logp_difference/max": 0.24903409481048583,
|
|
"sampling/sampling_logp_difference/mean": 0.021366760693490507,
|
|
"step": 8560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.7,
|
|
"completions/max_terminated_length": 290.7,
|
|
"completions/mean_length": 204.7625,
|
|
"completions/mean_terminated_length": 204.7625,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 1.0591071791946889,
|
|
"epoch": 0.7879010756642456,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.243019104003906,
|
|
"kl": 0.039074685383820904,
|
|
"learning_rate": 4.407216683166832e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 37266903.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.2692277133464813,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.2692277133464813,
|
|
"sampling/importance_sampling_ratio/max": 1.292909562587738,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000833988189697,
|
|
"sampling/importance_sampling_ratio/min": 0.7738879501819611,
|
|
"sampling/sampling_logp_difference/max": 0.2781393527984619,
|
|
"sampling/sampling_logp_difference/mean": 0.021012044697999954,
|
|
"step": 8570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.6,
|
|
"completions/max_terminated_length": 316.6,
|
|
"completions/mean_length": 236.5625,
|
|
"completions/mean_terminated_length": 236.5625,
|
|
"completions/min_length": 155.1,
|
|
"completions/min_terminated_length": 155.1,
|
|
"entropy": 1.0437770657241345,
|
|
"epoch": 0.788820446814379,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.6960649490356445,
|
|
"kl": 0.03853895569918677,
|
|
"learning_rate": 4.40557763431727e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 37314996.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.41183285415172577,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.4118328779935837,
|
|
"sampling/importance_sampling_ratio/max": 1.269897961616516,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999617874622345,
|
|
"sampling/importance_sampling_ratio/min": 0.7577203273773193,
|
|
"sampling/sampling_logp_difference/max": 0.287166428565979,
|
|
"sampling/sampling_logp_difference/mean": 0.020373632572591303,
|
|
"step": 8580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.7,
|
|
"completions/max_terminated_length": 302.7,
|
|
"completions/mean_length": 213.375,
|
|
"completions/mean_terminated_length": 213.375,
|
|
"completions/min_length": 139.5,
|
|
"completions/min_terminated_length": 139.5,
|
|
"entropy": 1.085709285736084,
|
|
"epoch": 0.7897398179645123,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.100148677825928,
|
|
"kl": 0.04313471909845248,
|
|
"learning_rate": 4.403936628286727e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 37358770.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.3734060972929001,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.37340610921382905,
|
|
"sampling/importance_sampling_ratio/max": 1.2621408581733704,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998152911663055,
|
|
"sampling/importance_sampling_ratio/min": 0.7815590083599091,
|
|
"sampling/sampling_logp_difference/max": 0.2683415412902832,
|
|
"sampling/sampling_logp_difference/mean": 0.021751171723008157,
|
|
"step": 8590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.0,
|
|
"completions/max_terminated_length": 277.0,
|
|
"completions/mean_length": 189.1875,
|
|
"completions/mean_terminated_length": 189.1875,
|
|
"completions/min_length": 120.5,
|
|
"completions/min_terminated_length": 120.5,
|
|
"entropy": 1.159361157566309,
|
|
"epoch": 0.7906591891146456,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.11463578790426254,
|
|
"kl": 0.030379978602286428,
|
|
"learning_rate": 4.402293666760648e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 37400577.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.4978660613298416,
|
|
"rewards/reward_fn/mean": 0.4125,
|
|
"rewards/reward_fn/std": 0.497866091132164,
|
|
"sampling/importance_sampling_ratio/max": 1.2752498269081116,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999164402484894,
|
|
"sampling/importance_sampling_ratio/min": 0.793759036064148,
|
|
"sampling/sampling_logp_difference/max": 0.27189621925354,
|
|
"sampling/sampling_logp_difference/mean": 0.022642544098198415,
|
|
"step": 8600
|
|
},
|
|
{
|
|
"epoch": 0.7906591891146456,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002399650959860384,
|
|
"eval_completions/max_length": 296.16579406631763,
|
|
"eval_completions/max_terminated_length": 293.4694589877836,
|
|
"eval_completions/mean_length": 208.75239965095986,
|
|
"eval_completions/mean_terminated_length": 208.05810917027125,
|
|
"eval_completions/min_length": 135.22513089005236,
|
|
"eval_completions/min_terminated_length": 135.22513089005236,
|
|
"eval_entropy": 1.089236964574035,
|
|
"eval_frac_reward_zero_std": 0.28970331588132636,
|
|
"eval_kl": 0.03935647829001799,
|
|
"eval_loss": 0.001974169397726655,
|
|
"eval_num_tokens": 37400577.0,
|
|
"eval_reward": 0.054864746945898775,
|
|
"eval_reward_std": 0.35963230924664574,
|
|
"eval_rewards/reward_fn/mean": 0.054864746945898775,
|
|
"eval_rewards/reward_fn/std": 0.35963231985689664,
|
|
"eval_runtime": 1731.2316,
|
|
"eval_samples_per_second": 0.331,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2659988542709883,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999380022235358,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7737978625568004,
|
|
"eval_sampling/sampling_logp_difference/max": 0.28144922056747357,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02118139997950347,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 8600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.1,
|
|
"completions/max_terminated_length": 301.1,
|
|
"completions/mean_length": 198.225,
|
|
"completions/mean_terminated_length": 198.225,
|
|
"completions/min_length": 120.5,
|
|
"completions/min_terminated_length": 120.5,
|
|
"entropy": 1.118923868983984,
|
|
"epoch": 0.7915785602647789,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.592360973358154,
|
|
"kl": 0.04915940761566162,
|
|
"learning_rate": 4.4006487514264855e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 37444339.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.30762176811695097,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.30762178599834444,
|
|
"sampling/importance_sampling_ratio/max": 1.2808159470558167,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999868094921112,
|
|
"sampling/importance_sampling_ratio/min": 0.7918745994567871,
|
|
"sampling/sampling_logp_difference/max": 0.26720079183578493,
|
|
"sampling/sampling_logp_difference/mean": 0.022386537864804268,
|
|
"step": 8610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 322.9,
|
|
"completions/max_terminated_length": 322.9,
|
|
"completions/mean_length": 230.6375,
|
|
"completions/mean_terminated_length": 230.6375,
|
|
"completions/min_length": 151.9,
|
|
"completions/min_terminated_length": 151.9,
|
|
"entropy": 1.0901392698287964,
|
|
"epoch": 0.7924979314149122,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.12871846556663513,
|
|
"kl": 0.036224769288674,
|
|
"learning_rate": 4.399001883973699e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 37494582.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.18515617549419403,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.185156187415123,
|
|
"sampling/importance_sampling_ratio/max": 1.2872913479804993,
|
|
"sampling/importance_sampling_ratio/mean": 1.000032478570938,
|
|
"sampling/importance_sampling_ratio/min": 0.765818428993225,
|
|
"sampling/sampling_logp_difference/max": 0.286179518699646,
|
|
"sampling/sampling_logp_difference/mean": 0.02144911829382181,
|
|
"step": 8620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.3,
|
|
"completions/max_terminated_length": 258.3,
|
|
"completions/mean_length": 197.9375,
|
|
"completions/mean_terminated_length": 197.9375,
|
|
"completions/min_length": 138.3,
|
|
"completions/min_terminated_length": 138.3,
|
|
"entropy": 1.0698160007596016,
|
|
"epoch": 0.7934173025650455,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 4.682737827301025,
|
|
"kl": 0.03345448941690847,
|
|
"learning_rate": 4.397353066093755e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 37538097.0,
|
|
"reward": -0.25,
|
|
"reward_std": 0.45998663604259493,
|
|
"rewards/reward_fn/mean": -0.25,
|
|
"rewards/reward_fn/std": 0.4599866420030594,
|
|
"sampling/importance_sampling_ratio/max": 1.233576202392578,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997283637523651,
|
|
"sampling/importance_sampling_ratio/min": 0.7977955639362335,
|
|
"sampling/sampling_logp_difference/max": 0.2359161615371704,
|
|
"sampling/sampling_logp_difference/mean": 0.02136957421898842,
|
|
"step": 8630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.8,
|
|
"completions/max_terminated_length": 324.8,
|
|
"completions/mean_length": 214.4125,
|
|
"completions/mean_terminated_length": 214.4125,
|
|
"completions/min_length": 140.4,
|
|
"completions/min_terminated_length": 140.4,
|
|
"entropy": 1.0040601417422295,
|
|
"epoch": 0.7943366737151788,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.951817512512207,
|
|
"kl": 0.04786549654672854,
|
|
"learning_rate": 4.39570229948012e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 37579002.0,
|
|
"reward": -0.3625,
|
|
"reward_std": 0.353536981344223,
|
|
"rewards/reward_fn/mean": -0.3625,
|
|
"rewards/reward_fn/std": 0.353536993265152,
|
|
"sampling/importance_sampling_ratio/max": 1.3053579568862914,
|
|
"sampling/importance_sampling_ratio/mean": 1.0010092914104463,
|
|
"sampling/importance_sampling_ratio/min": 0.7833929717540741,
|
|
"sampling/sampling_logp_difference/max": 0.2886221408843994,
|
|
"sampling/sampling_logp_difference/mean": 0.020288372226059437,
|
|
"step": 8640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.5,
|
|
"completions/max_terminated_length": 299.5,
|
|
"completions/mean_length": 209.6375,
|
|
"completions/mean_terminated_length": 209.6375,
|
|
"completions/min_length": 125.5,
|
|
"completions/min_terminated_length": 125.5,
|
|
"entropy": 1.0167246103286742,
|
|
"epoch": 0.7952560448653121,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.263909339904785,
|
|
"kl": 0.05170157368993387,
|
|
"learning_rate": 4.394049585828265e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 37621261.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.41830068826675415,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.41830070614814757,
|
|
"sampling/importance_sampling_ratio/max": 1.3136829137802124,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995709776878356,
|
|
"sampling/importance_sampling_ratio/min": 0.7284458369016648,
|
|
"sampling/sampling_logp_difference/max": 0.37468955516815183,
|
|
"sampling/sampling_logp_difference/mean": 0.02126057129353285,
|
|
"step": 8650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 353.4,
|
|
"completions/max_terminated_length": 353.4,
|
|
"completions/mean_length": 230.4125,
|
|
"completions/mean_terminated_length": 230.4125,
|
|
"completions/min_length": 148.9,
|
|
"completions/min_terminated_length": 148.9,
|
|
"entropy": 1.03956740796566,
|
|
"epoch": 0.7961754160154454,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.940287113189697,
|
|
"kl": 0.050082489562919365,
|
|
"learning_rate": 4.392394926835658e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 37665110.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.3409078598022461,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.3409078598022461,
|
|
"sampling/importance_sampling_ratio/max": 1.260983943939209,
|
|
"sampling/importance_sampling_ratio/mean": 0.999590802192688,
|
|
"sampling/importance_sampling_ratio/min": 0.8059561014175415,
|
|
"sampling/sampling_logp_difference/max": 0.2408542037010193,
|
|
"sampling/sampling_logp_difference/mean": 0.020256057381629944,
|
|
"step": 8660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.3,
|
|
"completions/max_terminated_length": 326.3,
|
|
"completions/mean_length": 226.2,
|
|
"completions/mean_terminated_length": 226.2,
|
|
"completions/min_length": 142.7,
|
|
"completions/min_terminated_length": 142.7,
|
|
"entropy": 1.09537383466959,
|
|
"epoch": 0.7970947871655787,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.319007873535156,
|
|
"kl": 0.05135476163122803,
|
|
"learning_rate": 4.390738324201767e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 37711510.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.37277381420135497,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.37277381420135497,
|
|
"sampling/importance_sampling_ratio/max": 1.289265477657318,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001185178756713,
|
|
"sampling/importance_sampling_ratio/min": 0.7453424364328385,
|
|
"sampling/sampling_logp_difference/max": 0.35207270085811615,
|
|
"sampling/sampling_logp_difference/mean": 0.021006496623158455,
|
|
"step": 8670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.5,
|
|
"completions/max_terminated_length": 301.5,
|
|
"completions/mean_length": 221.0,
|
|
"completions/mean_terminated_length": 221.0,
|
|
"completions/min_length": 138.3,
|
|
"completions/min_terminated_length": 138.3,
|
|
"entropy": 1.035111054778099,
|
|
"epoch": 0.7980141583157121,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.857322692871094,
|
|
"kl": 0.042267977393930775,
|
|
"learning_rate": 4.389079779628057e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 37758558.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.4968202650547028,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.4968202769756317,
|
|
"sampling/importance_sampling_ratio/max": 1.2256409406661988,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000116884708405,
|
|
"sampling/importance_sampling_ratio/min": 0.7691984653472901,
|
|
"sampling/sampling_logp_difference/max": 0.28101028203964235,
|
|
"sampling/sampling_logp_difference/mean": 0.020300885289907457,
|
|
"step": 8680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 287.2,
|
|
"completions/max_terminated_length": 278.2,
|
|
"completions/mean_length": 203.125,
|
|
"completions/mean_terminated_length": 200.90357360839843,
|
|
"completions/min_length": 119.8,
|
|
"completions/min_terminated_length": 119.8,
|
|
"entropy": 1.0503442004323005,
|
|
"epoch": 0.7989335294658454,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12584029138088226,
|
|
"kl": 0.04647665779921226,
|
|
"learning_rate": 4.3874192948179836e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 37801976.0,
|
|
"reward": 0.00625,
|
|
"reward_std": 0.45274376273155215,
|
|
"rewards/reward_fn/mean": 0.00625,
|
|
"rewards/reward_fn/std": 0.45274377465248106,
|
|
"sampling/importance_sampling_ratio/max": 1.2915782809257508,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004492461681367,
|
|
"sampling/importance_sampling_ratio/min": 0.7633813917636871,
|
|
"sampling/sampling_logp_difference/max": 0.32255802154541013,
|
|
"sampling/sampling_logp_difference/mean": 0.021532826870679856,
|
|
"step": 8690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.9,
|
|
"completions/max_terminated_length": 273.9,
|
|
"completions/mean_length": 209.0125,
|
|
"completions/mean_terminated_length": 209.0125,
|
|
"completions/min_length": 143.2,
|
|
"completions/min_terminated_length": 143.2,
|
|
"entropy": 1.0974105402827263,
|
|
"epoch": 0.7998529006159787,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.500042915344238,
|
|
"kl": 0.03251999069470912,
|
|
"learning_rate": 4.385756871477e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 37843193.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.22675940096378328,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.22675940692424773,
|
|
"sampling/importance_sampling_ratio/max": 1.2706981897354126,
|
|
"sampling/importance_sampling_ratio/mean": 1.00041384100914,
|
|
"sampling/importance_sampling_ratio/min": 0.7782668471336365,
|
|
"sampling/sampling_logp_difference/max": 0.2739842295646667,
|
|
"sampling/sampling_logp_difference/mean": 0.021877089887857436,
|
|
"step": 8700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 251.5,
|
|
"completions/max_terminated_length": 251.5,
|
|
"completions/mean_length": 183.5375,
|
|
"completions/mean_terminated_length": 183.5375,
|
|
"completions/min_length": 120.5,
|
|
"completions/min_terminated_length": 120.5,
|
|
"entropy": 1.075262963399291,
|
|
"epoch": 0.800772271766112,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.26422399282455444,
|
|
"kl": 0.04808458702173084,
|
|
"learning_rate": 4.384092511312547e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 37882668.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.353536981344223,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.353536993265152,
|
|
"sampling/importance_sampling_ratio/max": 1.2109286904335022,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996584713459015,
|
|
"sampling/importance_sampling_ratio/min": 0.7896126568317413,
|
|
"sampling/sampling_logp_difference/max": 0.24269652962684632,
|
|
"sampling/sampling_logp_difference/mean": 0.021072769351303578,
|
|
"step": 8710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.4,
|
|
"completions/max_terminated_length": 258.4,
|
|
"completions/mean_length": 188.4625,
|
|
"completions/mean_terminated_length": 188.4625,
|
|
"completions/min_length": 126.6,
|
|
"completions/min_terminated_length": 126.6,
|
|
"entropy": 0.9797229453921318,
|
|
"epoch": 0.8016916429162453,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.8295722007751465,
|
|
"kl": 0.0408133186516352,
|
|
"learning_rate": 4.382426216034057e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 37920569.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.44623149037361143,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.44623149037361143,
|
|
"sampling/importance_sampling_ratio/max": 1.2410557627677918,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003779232501984,
|
|
"sampling/importance_sampling_ratio/min": 0.7973136365413666,
|
|
"sampling/sampling_logp_difference/max": 0.2386075735092163,
|
|
"sampling/sampling_logp_difference/mean": 0.02035735584795475,
|
|
"step": 8720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.4,
|
|
"completions/max_terminated_length": 316.4,
|
|
"completions/mean_length": 224.5625,
|
|
"completions/mean_terminated_length": 224.5625,
|
|
"completions/min_length": 149.8,
|
|
"completions/min_terminated_length": 149.8,
|
|
"entropy": 1.047971683740616,
|
|
"epoch": 0.8026110140663786,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.982512474060059,
|
|
"kl": 0.05426092586712912,
|
|
"learning_rate": 4.380757987352948e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 37964982.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.3955159991979599,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.39551601111888884,
|
|
"sampling/importance_sampling_ratio/max": 1.3352445602416991,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002695739269256,
|
|
"sampling/importance_sampling_ratio/min": 0.7990143358707428,
|
|
"sampling/sampling_logp_difference/max": 0.291439151763916,
|
|
"sampling/sampling_logp_difference/mean": 0.02060706652700901,
|
|
"step": 8730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.1,
|
|
"completions/max_terminated_length": 289.1,
|
|
"completions/mean_length": 215.7875,
|
|
"completions/mean_terminated_length": 215.7875,
|
|
"completions/min_length": 138.3,
|
|
"completions/min_terminated_length": 138.3,
|
|
"entropy": 1.086515760421753,
|
|
"epoch": 0.8035303852165119,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.220156192779541,
|
|
"kl": 0.040553272410761565,
|
|
"learning_rate": 4.379087826982625e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 38010005.0,
|
|
"reward": 0.19375,
|
|
"reward_std": 0.3894675612449646,
|
|
"rewards/reward_fn/mean": 0.19375,
|
|
"rewards/reward_fn/std": 0.3894675672054291,
|
|
"sampling/importance_sampling_ratio/max": 1.2983322143554688,
|
|
"sampling/importance_sampling_ratio/mean": 0.999848073720932,
|
|
"sampling/importance_sampling_ratio/min": 0.781996202468872,
|
|
"sampling/sampling_logp_difference/max": 0.2927091121673584,
|
|
"sampling/sampling_logp_difference/mean": 0.021740529499948025,
|
|
"step": 8740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 336.3,
|
|
"completions/max_terminated_length": 336.3,
|
|
"completions/mean_length": 221.6125,
|
|
"completions/mean_terminated_length": 221.6125,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.0934100411832333,
|
|
"epoch": 0.8044497563666452,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.16055728495121002,
|
|
"kl": 0.047162325971294196,
|
|
"learning_rate": 4.377415736638476e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 38052566.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.19864802658557892,
|
|
"sampling/importance_sampling_ratio/max": 1.2874309778213502,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998765051364898,
|
|
"sampling/importance_sampling_ratio/min": 0.7950535893440247,
|
|
"sampling/sampling_logp_difference/max": 0.27124361991882323,
|
|
"sampling/sampling_logp_difference/mean": 0.02178285215049982,
|
|
"step": 8750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.7,
|
|
"completions/max_terminated_length": 278.7,
|
|
"completions/mean_length": 194.75,
|
|
"completions/mean_terminated_length": 194.75,
|
|
"completions/min_length": 121.1,
|
|
"completions/min_terminated_length": 121.1,
|
|
"entropy": 1.0251334503293037,
|
|
"epoch": 0.8053691275167785,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.286441326141357,
|
|
"kl": 0.04760275386506692,
|
|
"learning_rate": 4.375741718037873e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 38091882.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.1632926881313324,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.1632926881313324,
|
|
"sampling/importance_sampling_ratio/max": 1.2806723594665528,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000867605209351,
|
|
"sampling/importance_sampling_ratio/min": 0.7584055721759796,
|
|
"sampling/sampling_logp_difference/max": 0.2932913899421692,
|
|
"sampling/sampling_logp_difference/mean": 0.02150655761361122,
|
|
"step": 8760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 303.3,
|
|
"completions/max_terminated_length": 286.5,
|
|
"completions/mean_length": 224.5125,
|
|
"completions/mean_terminated_length": 216.3375,
|
|
"completions/min_length": 144.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"entropy": 1.1176640853285789,
|
|
"epoch": 0.8062884986669119,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1579965353012085,
|
|
"kl": 0.0352932273526676,
|
|
"learning_rate": 4.374065772900165e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 38138163.0,
|
|
"reward": 0.19375,
|
|
"reward_std": 0.48904638886451723,
|
|
"rewards/reward_fn/mean": 0.19375,
|
|
"rewards/reward_fn/std": 0.4890464127063751,
|
|
"sampling/importance_sampling_ratio/max": 1.2696520686149597,
|
|
"sampling/importance_sampling_ratio/mean": 0.999466073513031,
|
|
"sampling/importance_sampling_ratio/min": 0.7751040577888488,
|
|
"sampling/sampling_logp_difference/max": 0.265606153011322,
|
|
"sampling/sampling_logp_difference/mean": 0.02191924545913935,
|
|
"step": 8770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.4,
|
|
"completions/max_terminated_length": 304.4,
|
|
"completions/mean_length": 190.925,
|
|
"completions/mean_terminated_length": 190.925,
|
|
"completions/min_length": 116.5,
|
|
"completions/min_terminated_length": 116.5,
|
|
"entropy": 1.0450495783239604,
|
|
"epoch": 0.8072078698170452,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.599582195281982,
|
|
"kl": 0.05343142842175439,
|
|
"learning_rate": 4.372387902946684e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 38182573.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.2667276442050934,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.2667276501655579,
|
|
"sampling/importance_sampling_ratio/max": 1.2608490586280823,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005966007709504,
|
|
"sampling/importance_sampling_ratio/min": 0.7757447600364685,
|
|
"sampling/sampling_logp_difference/max": 0.2792081117630005,
|
|
"sampling/sampling_logp_difference/mean": 0.02111231945455074,
|
|
"step": 8780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.9,
|
|
"completions/max_terminated_length": 277.9,
|
|
"completions/mean_length": 200.3375,
|
|
"completions/mean_terminated_length": 200.3375,
|
|
"completions/min_length": 129.4,
|
|
"completions/min_terminated_length": 129.4,
|
|
"entropy": 1.0650831572711468,
|
|
"epoch": 0.8081272409671785,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.140339374542236,
|
|
"kl": 0.0357040181173943,
|
|
"learning_rate": 4.370708109900737e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 38222272.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4363411724567413,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.4363411784172058,
|
|
"sampling/importance_sampling_ratio/max": 1.2859436869621277,
|
|
"sampling/importance_sampling_ratio/mean": 0.999999725818634,
|
|
"sampling/importance_sampling_ratio/min": 0.7639418482780457,
|
|
"sampling/sampling_logp_difference/max": 0.288063645362854,
|
|
"sampling/sampling_logp_difference/mean": 0.021699277497828007,
|
|
"step": 8790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 259.0,
|
|
"completions/max_terminated_length": 259.0,
|
|
"completions/mean_length": 187.625,
|
|
"completions/mean_terminated_length": 187.625,
|
|
"completions/min_length": 135.9,
|
|
"completions/min_terminated_length": 135.9,
|
|
"entropy": 0.9379396364092827,
|
|
"epoch": 0.8090466121173118,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.193876266479492,
|
|
"kl": 0.04675668946583755,
|
|
"learning_rate": 4.369026395487605e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 38262714.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.34817772209644315,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.34817772805690766,
|
|
"sampling/importance_sampling_ratio/max": 1.3576258540153503,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003045678138733,
|
|
"sampling/importance_sampling_ratio/min": 0.707453191280365,
|
|
"sampling/sampling_logp_difference/max": 0.3967010617256165,
|
|
"sampling/sampling_logp_difference/mean": 0.02055827286094427,
|
|
"step": 8800
|
|
},
|
|
{
|
|
"epoch": 0.8090466121173118,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0015270506108202443,
|
|
"eval_completions/max_length": 287.2844677137871,
|
|
"eval_completions/max_terminated_length": 286.1675392670157,
|
|
"eval_completions/mean_length": 203.38852530541013,
|
|
"eval_completions/mean_terminated_length": 203.06027192387074,
|
|
"eval_completions/min_length": 130.64746945898779,
|
|
"eval_completions/min_terminated_length": 130.64746945898779,
|
|
"eval_entropy": 1.080786794684112,
|
|
"eval_frac_reward_zero_std": 0.32460732984293195,
|
|
"eval_kl": 0.04283483098317945,
|
|
"eval_loss": 0.002224038355052471,
|
|
"eval_num_tokens": 38262714.0,
|
|
"eval_reward": 0.05475567190226876,
|
|
"eval_reward_std": 0.35207359607203054,
|
|
"eval_rewards/reward_fn/mean": 0.05475567190226876,
|
|
"eval_rewards/reward_fn/std": 0.35207360595412696,
|
|
"eval_runtime": 1684.7806,
|
|
"eval_samples_per_second": 0.34,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2736616600662418,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000273790242993,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7780227620564206,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2795751323666664,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021095979768607317,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 8800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.5,
|
|
"completions/max_terminated_length": 296.5,
|
|
"completions/mean_length": 211.525,
|
|
"completions/mean_terminated_length": 211.525,
|
|
"completions/min_length": 136.1,
|
|
"completions/min_terminated_length": 136.1,
|
|
"entropy": 1.0855183489620686,
|
|
"epoch": 0.8099659832674451,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.21091626584529877,
|
|
"kl": 0.03973880144767463,
|
|
"learning_rate": 4.367342761434544e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 38307772.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.3818357288837433,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.38183574080467225,
|
|
"sampling/importance_sampling_ratio/max": 1.2601247310638428,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001230716705323,
|
|
"sampling/importance_sampling_ratio/min": 0.7872125804424286,
|
|
"sampling/sampling_logp_difference/max": 0.26520882844924926,
|
|
"sampling/sampling_logp_difference/mean": 0.021789366938173772,
|
|
"step": 8810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.2,
|
|
"completions/max_terminated_length": 291.2,
|
|
"completions/mean_length": 210.975,
|
|
"completions/mean_terminated_length": 210.975,
|
|
"completions/min_length": 131.2,
|
|
"completions/min_terminated_length": 131.2,
|
|
"entropy": 1.0969344601035118,
|
|
"epoch": 0.8108853544175784,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.790411949157715,
|
|
"kl": 0.04803976528928615,
|
|
"learning_rate": 4.365657209470781e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 38351874.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.4075396120548248,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.4075396180152893,
|
|
"sampling/importance_sampling_ratio/max": 1.2696874022483826,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002799391746522,
|
|
"sampling/importance_sampling_ratio/min": 0.7690694212913514,
|
|
"sampling/sampling_logp_difference/max": 0.28398908376693727,
|
|
"sampling/sampling_logp_difference/mean": 0.02169826366007328,
|
|
"step": 8820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 292.0,
|
|
"completions/max_terminated_length": 282.1,
|
|
"completions/mean_length": 206.2125,
|
|
"completions/mean_terminated_length": 202.51607208251954,
|
|
"completions/min_length": 136.5,
|
|
"completions/min_terminated_length": 136.5,
|
|
"entropy": 1.0781104639172554,
|
|
"epoch": 0.8118047255677117,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.09663721174001694,
|
|
"kl": 0.028700974106322975,
|
|
"learning_rate": 4.363969741327514e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 38391947.0,
|
|
"reward": 0.44375,
|
|
"reward_std": 0.31051755249500274,
|
|
"rewards/reward_fn/mean": 0.44375,
|
|
"rewards/reward_fn/std": 0.31051755249500274,
|
|
"sampling/importance_sampling_ratio/max": 1.2981590628623962,
|
|
"sampling/importance_sampling_ratio/mean": 1.000667291879654,
|
|
"sampling/importance_sampling_ratio/min": 0.7828846096992492,
|
|
"sampling/sampling_logp_difference/max": 0.2753793835639954,
|
|
"sampling/sampling_logp_difference/mean": 0.021466749347746374,
|
|
"step": 8830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.3,
|
|
"completions/max_terminated_length": 272.3,
|
|
"completions/mean_length": 190.85,
|
|
"completions/mean_terminated_length": 190.85,
|
|
"completions/min_length": 124.4,
|
|
"completions/min_terminated_length": 124.4,
|
|
"entropy": 1.1052660129964351,
|
|
"epoch": 0.812724096717845,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.288593292236328,
|
|
"kl": 0.03749731448478997,
|
|
"learning_rate": 4.362280358737907e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 38431855.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.43029355704784394,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.4302935689687729,
|
|
"sampling/importance_sampling_ratio/max": 1.254835045337677,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001147627830504,
|
|
"sampling/importance_sampling_ratio/min": 0.7889634609222412,
|
|
"sampling/sampling_logp_difference/max": 0.26200541853904724,
|
|
"sampling/sampling_logp_difference/mean": 0.02180521059781313,
|
|
"step": 8840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.1,
|
|
"completions/max_terminated_length": 278.1,
|
|
"completions/mean_length": 196.85,
|
|
"completions/mean_terminated_length": 196.85,
|
|
"completions/min_length": 133.1,
|
|
"completions/min_terminated_length": 133.1,
|
|
"entropy": 1.0610683955252171,
|
|
"epoch": 0.8136434678679783,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.09803583472967148,
|
|
"kl": 0.03152940608561039,
|
|
"learning_rate": 4.360589063437091e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 38472763.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.27396377325057986,
|
|
"sampling/importance_sampling_ratio/max": 1.2634969353675842,
|
|
"sampling/importance_sampling_ratio/mean": 1.000402319431305,
|
|
"sampling/importance_sampling_ratio/min": 0.7855935394763947,
|
|
"sampling/sampling_logp_difference/max": 0.2712444543838501,
|
|
"sampling/sampling_logp_difference/mean": 0.020579307712614535,
|
|
"step": 8850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.3,
|
|
"completions/max_terminated_length": 280.3,
|
|
"completions/mean_length": 209.775,
|
|
"completions/mean_terminated_length": 209.775,
|
|
"completions/min_length": 144.1,
|
|
"completions/min_terminated_length": 144.1,
|
|
"entropy": 1.0386340081691743,
|
|
"epoch": 0.8145628390181117,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.450444221496582,
|
|
"kl": 0.038624640996567906,
|
|
"learning_rate": 4.358895857162163e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 38515385.0,
|
|
"reward": 0.41875,
|
|
"reward_std": 0.3511351436376572,
|
|
"rewards/reward_fn/mean": 0.41875,
|
|
"rewards/reward_fn/std": 0.3511351615190506,
|
|
"sampling/importance_sampling_ratio/max": 1.2828514575958252,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003724694252014,
|
|
"sampling/importance_sampling_ratio/min": 0.7626035332679748,
|
|
"sampling/sampling_logp_difference/max": 0.28701198697090147,
|
|
"sampling/sampling_logp_difference/mean": 0.021290628984570504,
|
|
"step": 8860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 315.9,
|
|
"completions/max_terminated_length": 304.2,
|
|
"completions/mean_length": 224.3375,
|
|
"completions/mean_terminated_length": 221.375,
|
|
"completions/min_length": 138.5,
|
|
"completions/min_terminated_length": 138.5,
|
|
"entropy": 1.097715050727129,
|
|
"epoch": 0.815482210168245,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12754617631435394,
|
|
"kl": 0.041775132965995,
|
|
"learning_rate": 4.35720074165218e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 38557692.0,
|
|
"reward": -0.14375,
|
|
"reward_std": 0.42964828312397,
|
|
"rewards/reward_fn/mean": -0.14375,
|
|
"rewards/reward_fn/std": 0.42964828312397,
|
|
"sampling/importance_sampling_ratio/max": 1.2867362260818482,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999699056148529,
|
|
"sampling/importance_sampling_ratio/min": 0.786135321855545,
|
|
"sampling/sampling_logp_difference/max": 0.28501527309417723,
|
|
"sampling/sampling_logp_difference/mean": 0.022443701326847077,
|
|
"step": 8870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.6,
|
|
"completions/max_terminated_length": 273.6,
|
|
"completions/mean_length": 202.35,
|
|
"completions/mean_terminated_length": 202.35,
|
|
"completions/min_length": 144.3,
|
|
"completions/min_terminated_length": 144.3,
|
|
"entropy": 0.9785729929804802,
|
|
"epoch": 0.8164015813183783,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.09973826259374619,
|
|
"kl": 0.036937700293492526,
|
|
"learning_rate": 4.355503718648163e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 38599312.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.23484184443950654,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.23484184443950654,
|
|
"sampling/importance_sampling_ratio/max": 1.3120640397071839,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001800775527954,
|
|
"sampling/importance_sampling_ratio/min": 0.7991429746150971,
|
|
"sampling/sampling_logp_difference/max": 0.28285343647003175,
|
|
"sampling/sampling_logp_difference/mean": 0.020083884708583355,
|
|
"step": 8880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.6,
|
|
"completions/max_terminated_length": 293.6,
|
|
"completions/mean_length": 209.275,
|
|
"completions/mean_terminated_length": 209.275,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 1.0875718615949155,
|
|
"epoch": 0.8173209524685116,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.550856113433838,
|
|
"kl": 0.037851628882344814,
|
|
"learning_rate": 4.35380478989309e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 38643430.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.3886036962270737,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.3886037141084671,
|
|
"sampling/importance_sampling_ratio/max": 1.3095089793205261,
|
|
"sampling/importance_sampling_ratio/mean": 1.0009268760681151,
|
|
"sampling/importance_sampling_ratio/min": 0.7880194306373596,
|
|
"sampling/sampling_logp_difference/max": 0.2841110944747925,
|
|
"sampling/sampling_logp_difference/mean": 0.021922591514885424,
|
|
"step": 8890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 345.5,
|
|
"completions/max_terminated_length": 323.3,
|
|
"completions/mean_length": 234.325,
|
|
"completions/mean_terminated_length": 230.87857208251953,
|
|
"completions/min_length": 159.2,
|
|
"completions/min_terminated_length": 159.2,
|
|
"entropy": 1.0169524781405925,
|
|
"epoch": 0.8182403236186448,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.433189868927002,
|
|
"kl": 0.04369154053274542,
|
|
"learning_rate": 4.352103957131898e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 38689392.0,
|
|
"reward": -0.18125,
|
|
"reward_std": 0.47442744076251986,
|
|
"rewards/reward_fn/mean": -0.18125,
|
|
"rewards/reward_fn/std": 0.4744274467229843,
|
|
"sampling/importance_sampling_ratio/max": 1.2639684796333313,
|
|
"sampling/importance_sampling_ratio/mean": 1.000538033246994,
|
|
"sampling/importance_sampling_ratio/min": 0.7510909378528595,
|
|
"sampling/sampling_logp_difference/max": 0.2981226921081543,
|
|
"sampling/sampling_logp_difference/mean": 0.02065893579274416,
|
|
"step": 8900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.8,
|
|
"completions/max_terminated_length": 314.8,
|
|
"completions/mean_length": 212.9,
|
|
"completions/mean_terminated_length": 212.9,
|
|
"completions/min_length": 140.9,
|
|
"completions/min_terminated_length": 140.9,
|
|
"entropy": 1.0175654284656048,
|
|
"epoch": 0.8191596947687781,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.11694075912237167,
|
|
"kl": 0.04509069290943444,
|
|
"learning_rate": 4.350401222111477e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 38733976.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.3587038516998291,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.3587038516998291,
|
|
"sampling/importance_sampling_ratio/max": 1.2507014632225038,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001662015914916,
|
|
"sampling/importance_sampling_ratio/min": 0.7634660482406617,
|
|
"sampling/sampling_logp_difference/max": 0.2968484401702881,
|
|
"sampling/sampling_logp_difference/mean": 0.020928441546857357,
|
|
"step": 8910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 365.1,
|
|
"completions/max_terminated_length": 351.3,
|
|
"completions/mean_length": 258.7625,
|
|
"completions/mean_terminated_length": 250.1482147216797,
|
|
"completions/min_length": 163.1,
|
|
"completions/min_terminated_length": 163.1,
|
|
"entropy": 1.1511157929897309,
|
|
"epoch": 0.8200790659189114,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.27891287207603455,
|
|
"kl": 0.0469004514219705,
|
|
"learning_rate": 4.348696586580673e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 38783205.0,
|
|
"reward": -0.05625,
|
|
"reward_std": 0.4334175020456314,
|
|
"rewards/reward_fn/mean": -0.05625,
|
|
"rewards/reward_fn/std": 0.43341751396656036,
|
|
"sampling/importance_sampling_ratio/max": 1.2654539585113525,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997540295124054,
|
|
"sampling/importance_sampling_ratio/min": 0.7700287818908691,
|
|
"sampling/sampling_logp_difference/max": 0.2753345608711243,
|
|
"sampling/sampling_logp_difference/mean": 0.021777121350169182,
|
|
"step": 8920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.1,
|
|
"completions/max_terminated_length": 289.1,
|
|
"completions/mean_length": 208.6125,
|
|
"completions/mean_terminated_length": 208.6125,
|
|
"completions/min_length": 134.8,
|
|
"completions/min_terminated_length": 134.8,
|
|
"entropy": 1.0555961526930333,
|
|
"epoch": 0.8209984370690447,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.751936912536621,
|
|
"kl": 0.04429230668465607,
|
|
"learning_rate": 4.346990052290283e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 38824446.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.269358703494072,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.269358703494072,
|
|
"sampling/importance_sampling_ratio/max": 1.2969286561012268,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997332870960236,
|
|
"sampling/importance_sampling_ratio/min": 0.7328262656927109,
|
|
"sampling/sampling_logp_difference/max": 0.3552848696708679,
|
|
"sampling/sampling_logp_difference/mean": 0.021560726501047612,
|
|
"step": 8930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.7,
|
|
"completions/max_terminated_length": 274.7,
|
|
"completions/mean_length": 208.025,
|
|
"completions/mean_terminated_length": 208.025,
|
|
"completions/min_length": 142.3,
|
|
"completions/min_terminated_length": 142.3,
|
|
"entropy": 1.078001006692648,
|
|
"epoch": 0.821917808219178,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.077641010284424,
|
|
"kl": 0.04658143932465464,
|
|
"learning_rate": 4.345281620993056e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 38867728.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.4007413417100906,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.40074135959148405,
|
|
"sampling/importance_sampling_ratio/max": 1.2712863445281983,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997042357921601,
|
|
"sampling/importance_sampling_ratio/min": 0.7841499507427215,
|
|
"sampling/sampling_logp_difference/max": 0.2648874998092651,
|
|
"sampling/sampling_logp_difference/mean": 0.021290831640362738,
|
|
"step": 8940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.9,
|
|
"completions/max_terminated_length": 291.9,
|
|
"completions/mean_length": 215.6875,
|
|
"completions/mean_terminated_length": 215.6875,
|
|
"completions/min_length": 155.6,
|
|
"completions/min_terminated_length": 155.6,
|
|
"entropy": 1.0468795076012611,
|
|
"epoch": 0.8228371793693114,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12104776501655579,
|
|
"kl": 0.04860833355924114,
|
|
"learning_rate": 4.343571294443686e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 38910031.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.3472063720226288,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.3472063958644867,
|
|
"sampling/importance_sampling_ratio/max": 1.3149312734603882,
|
|
"sampling/importance_sampling_ratio/mean": 1.000365149974823,
|
|
"sampling/importance_sampling_ratio/min": 0.7600172936916352,
|
|
"sampling/sampling_logp_difference/max": 0.30601245164871216,
|
|
"sampling/sampling_logp_difference/mean": 0.02072648238390684,
|
|
"step": 8950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.8,
|
|
"completions/max_terminated_length": 272.8,
|
|
"completions/mean_length": 198.4125,
|
|
"completions/mean_terminated_length": 198.4125,
|
|
"completions/min_length": 125.2,
|
|
"completions/min_terminated_length": 125.2,
|
|
"entropy": 1.0973112799227238,
|
|
"epoch": 0.8237565505194447,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.14992687106132507,
|
|
"kl": 0.05303641356294975,
|
|
"learning_rate": 4.3418590743988164e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 38952352.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.18515617549419403,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.185156187415123,
|
|
"sampling/importance_sampling_ratio/max": 1.2804394125938416,
|
|
"sampling/importance_sampling_ratio/mean": 1.000007700920105,
|
|
"sampling/importance_sampling_ratio/min": 0.7859462320804596,
|
|
"sampling/sampling_logp_difference/max": 0.27508476972579954,
|
|
"sampling/sampling_logp_difference/mean": 0.021746984496712685,
|
|
"step": 8960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 330.4,
|
|
"completions/max_terminated_length": 317.9,
|
|
"completions/mean_length": 234.8625,
|
|
"completions/mean_terminated_length": 232.4214294433594,
|
|
"completions/min_length": 159.2,
|
|
"completions/min_terminated_length": 159.2,
|
|
"entropy": 1.162705421447754,
|
|
"epoch": 0.824675921669578,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12086024880409241,
|
|
"kl": 0.040746063087135555,
|
|
"learning_rate": 4.340144962617034e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 39003229.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.32662735879421234,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.3266273647546768,
|
|
"sampling/importance_sampling_ratio/max": 1.2446288347244263,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993978559970855,
|
|
"sampling/importance_sampling_ratio/min": 0.788626766204834,
|
|
"sampling/sampling_logp_difference/max": 0.2591118335723877,
|
|
"sampling/sampling_logp_difference/mean": 0.021541816368699073,
|
|
"step": 8970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.7,
|
|
"completions/max_terminated_length": 276.7,
|
|
"completions/mean_length": 207.0875,
|
|
"completions/mean_terminated_length": 207.0875,
|
|
"completions/min_length": 135.8,
|
|
"completions/min_terminated_length": 135.8,
|
|
"entropy": 1.1071314468979836,
|
|
"epoch": 0.8255952928197113,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.019286155700684,
|
|
"kl": 0.05420791409560479,
|
|
"learning_rate": 4.33842896085887e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 39047012.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.36287996768951414,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.3628799855709076,
|
|
"sampling/importance_sampling_ratio/max": 1.2749491333961487,
|
|
"sampling/importance_sampling_ratio/mean": 1.000269103050232,
|
|
"sampling/importance_sampling_ratio/min": 0.7728374063968658,
|
|
"sampling/sampling_logp_difference/max": 0.278046977519989,
|
|
"sampling/sampling_logp_difference/mean": 0.021776656992733477,
|
|
"step": 8980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 333.0,
|
|
"completions/max_terminated_length": 333.0,
|
|
"completions/mean_length": 232.55,
|
|
"completions/mean_terminated_length": 232.55,
|
|
"completions/min_length": 147.1,
|
|
"completions/min_terminated_length": 147.1,
|
|
"entropy": 1.0654337860643863,
|
|
"epoch": 0.8265146639698446,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 6.427915096282959,
|
|
"kl": 0.03480404880247079,
|
|
"learning_rate": 4.336711070886795e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 39095592.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.18515617549419403,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.185156187415123,
|
|
"sampling/importance_sampling_ratio/max": 1.2899380803108216,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001848757266998,
|
|
"sampling/importance_sampling_ratio/min": 0.7713256061077118,
|
|
"sampling/sampling_logp_difference/max": 0.27633455991744993,
|
|
"sampling/sampling_logp_difference/mean": 0.021645659022033213,
|
|
"step": 8990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.2,
|
|
"completions/max_terminated_length": 276.2,
|
|
"completions/mean_length": 209.6375,
|
|
"completions/mean_terminated_length": 209.6375,
|
|
"completions/min_length": 142.6,
|
|
"completions/min_terminated_length": 142.6,
|
|
"entropy": 1.0761123105883599,
|
|
"epoch": 0.8274340351199779,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.605615139007568,
|
|
"kl": 0.06172620209399611,
|
|
"learning_rate": 4.33499129446522e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 39140587.0,
|
|
"reward": 0.11875,
|
|
"reward_std": 0.36787369549274446,
|
|
"rewards/reward_fn/mean": 0.11875,
|
|
"rewards/reward_fn/std": 0.36787369549274446,
|
|
"sampling/importance_sampling_ratio/max": 1.2738475918769836,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001766085624695,
|
|
"sampling/importance_sampling_ratio/min": 0.7630397915840149,
|
|
"sampling/sampling_logp_difference/max": 0.2871168553829193,
|
|
"sampling/sampling_logp_difference/mean": 0.02122314050793648,
|
|
"step": 9000
|
|
},
|
|
{
|
|
"epoch": 0.8274340351199779,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 286.66841186736474,
|
|
"eval_completions/max_terminated_length": 285.5410122164049,
|
|
"eval_completions/mean_length": 204.09707678883072,
|
|
"eval_completions/mean_terminated_length": 203.72582698908568,
|
|
"eval_completions/min_length": 135.1849912739965,
|
|
"eval_completions/min_terminated_length": 135.1849912739965,
|
|
"eval_entropy": 1.0882200900900842,
|
|
"eval_frac_reward_zero_std": 0.38917975567190227,
|
|
"eval_kl": 0.0510732689726103,
|
|
"eval_loss": 0.0026241466403007507,
|
|
"eval_num_tokens": 39140587.0,
|
|
"eval_reward": 0.07340750436300175,
|
|
"eval_reward_std": 0.2887389008443393,
|
|
"eval_rewards/reward_fn/mean": 0.07340750436300175,
|
|
"eval_rewards/reward_fn/std": 0.28873890947817094,
|
|
"eval_runtime": 1681.4939,
|
|
"eval_samples_per_second": 0.341,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.267262753391765,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000146895803081,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7786260428749038,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2788939066166237,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020763859975218357,
|
|
"eval_steps_per_second": 0.043,
|
|
"step": 9000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.7,
|
|
"completions/max_terminated_length": 315.7,
|
|
"completions/mean_length": 211.225,
|
|
"completions/mean_terminated_length": 211.225,
|
|
"completions/min_length": 120.5,
|
|
"completions/min_terminated_length": 120.5,
|
|
"entropy": 1.1148918487131596,
|
|
"epoch": 0.8283534062701112,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.0368123054504395,
|
|
"kl": 0.057120157731696965,
|
|
"learning_rate": 4.333269633360493e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 39184709.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.31448993980884554,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.31448994576931,
|
|
"sampling/importance_sampling_ratio/max": 1.3122798442840575,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004422128200532,
|
|
"sampling/importance_sampling_ratio/min": 0.7739920496940613,
|
|
"sampling/sampling_logp_difference/max": 0.30198254585266116,
|
|
"sampling/sampling_logp_difference/mean": 0.021951787732541563,
|
|
"step": 9010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.1,
|
|
"completions/max_terminated_length": 278.1,
|
|
"completions/mean_length": 204.85,
|
|
"completions/mean_terminated_length": 204.85,
|
|
"completions/min_length": 140.4,
|
|
"completions/min_terminated_length": 140.4,
|
|
"entropy": 1.0834312930703163,
|
|
"epoch": 0.8292727774202445,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.699468612670898,
|
|
"kl": 0.06076372329844162,
|
|
"learning_rate": 4.3315460893408975e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 39228633.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.31131347119808195,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.3113134950399399,
|
|
"sampling/importance_sampling_ratio/max": 1.2907068371772765,
|
|
"sampling/importance_sampling_ratio/mean": 1.000281608104706,
|
|
"sampling/importance_sampling_ratio/min": 0.7726169645786285,
|
|
"sampling/sampling_logp_difference/max": 0.27844642400741576,
|
|
"sampling/sampling_logp_difference/mean": 0.021578053943812847,
|
|
"step": 9020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 321.3,
|
|
"completions/max_terminated_length": 321.3,
|
|
"completions/mean_length": 219.55,
|
|
"completions/mean_terminated_length": 219.55,
|
|
"completions/min_length": 140.4,
|
|
"completions/min_terminated_length": 140.4,
|
|
"entropy": 1.1024599201977252,
|
|
"epoch": 0.8301921485703778,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.15127870440483093,
|
|
"kl": 0.046587085886858406,
|
|
"learning_rate": 4.329820664176651e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 39273653.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.33288776576519014,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.3328877717256546,
|
|
"sampling/importance_sampling_ratio/max": 1.2989366888999938,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997116506099701,
|
|
"sampling/importance_sampling_ratio/min": 0.7541512727737427,
|
|
"sampling/sampling_logp_difference/max": 0.30350762605667114,
|
|
"sampling/sampling_logp_difference/mean": 0.02134474255144596,
|
|
"step": 9030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.2,
|
|
"completions/max_terminated_length": 325.2,
|
|
"completions/mean_length": 232.3125,
|
|
"completions/mean_terminated_length": 232.3125,
|
|
"completions/min_length": 137.5,
|
|
"completions/min_terminated_length": 137.5,
|
|
"entropy": 1.0963632196187973,
|
|
"epoch": 0.8311115197205111,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.10404127836227417,
|
|
"kl": 0.05548367394949309,
|
|
"learning_rate": 4.3280933596399066e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 39324646.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.20958369374275207,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.20958369374275207,
|
|
"sampling/importance_sampling_ratio/max": 1.2953606605529786,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003344476222993,
|
|
"sampling/importance_sampling_ratio/min": 0.774643874168396,
|
|
"sampling/sampling_logp_difference/max": 0.2935644745826721,
|
|
"sampling/sampling_logp_difference/mean": 0.02066124528646469,
|
|
"step": 9040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 339.7,
|
|
"completions/max_terminated_length": 339.7,
|
|
"completions/mean_length": 224.3125,
|
|
"completions/mean_terminated_length": 224.3125,
|
|
"completions/min_length": 140.0,
|
|
"completions/min_terminated_length": 140.0,
|
|
"entropy": 1.0464645341038703,
|
|
"epoch": 0.8320308908706445,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.13783994317054749,
|
|
"kl": 0.0407357525837142,
|
|
"learning_rate": 4.326364177504741e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 39371071.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.33679819107055664,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.3367981970310211,
|
|
"sampling/importance_sampling_ratio/max": 1.254710328578949,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000968396663665,
|
|
"sampling/importance_sampling_ratio/min": 0.7936439573764801,
|
|
"sampling/sampling_logp_difference/max": 0.24865182638168334,
|
|
"sampling/sampling_logp_difference/mean": 0.020618128962814807,
|
|
"step": 9050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.9,
|
|
"completions/max_terminated_length": 293.9,
|
|
"completions/mean_length": 205.1,
|
|
"completions/mean_terminated_length": 205.1,
|
|
"completions/min_length": 131.1,
|
|
"completions/min_terminated_length": 131.1,
|
|
"entropy": 1.1125460632145405,
|
|
"epoch": 0.8329502620207778,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.1732306331396103,
|
|
"kl": 0.048045880493009466,
|
|
"learning_rate": 4.324633119547163e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 39413471.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.13479767441749574,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.13479767441749574,
|
|
"sampling/importance_sampling_ratio/max": 1.2365493774414062,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004522264003755,
|
|
"sampling/importance_sampling_ratio/min": 0.7927963078022003,
|
|
"sampling/sampling_logp_difference/max": 0.24077730178833007,
|
|
"sampling/sampling_logp_difference/mean": 0.02182335276156664,
|
|
"step": 9060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 246.4,
|
|
"completions/max_terminated_length": 246.4,
|
|
"completions/mean_length": 177.0625,
|
|
"completions/mean_terminated_length": 177.0625,
|
|
"completions/min_length": 112.4,
|
|
"completions/min_terminated_length": 112.4,
|
|
"entropy": 1.0748204439878464,
|
|
"epoch": 0.8338696331709111,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.4584197998046875,
|
|
"kl": 0.038400100072612985,
|
|
"learning_rate": 4.32290018754511e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 39454700.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.3502548158168793,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.35025482773780825,
|
|
"sampling/importance_sampling_ratio/max": 1.3449970126152038,
|
|
"sampling/importance_sampling_ratio/mean": 1.0013143837451934,
|
|
"sampling/importance_sampling_ratio/min": 0.7819990158081055,
|
|
"sampling/sampling_logp_difference/max": 0.32233076095581054,
|
|
"sampling/sampling_logp_difference/mean": 0.021630468592047692,
|
|
"step": 9070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.7,
|
|
"completions/max_terminated_length": 275.7,
|
|
"completions/mean_length": 186.75,
|
|
"completions/mean_terminated_length": 186.75,
|
|
"completions/min_length": 112.4,
|
|
"completions/min_terminated_length": 112.4,
|
|
"entropy": 1.0550425782799722,
|
|
"epoch": 0.8347890043210444,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.490652084350586,
|
|
"kl": 0.030567353643709792,
|
|
"learning_rate": 4.3211653832784415e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 39491600.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.28198385536670684,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.28198386132717135,
|
|
"sampling/importance_sampling_ratio/max": 1.2257599234580994,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004043102264404,
|
|
"sampling/importance_sampling_ratio/min": 0.7733115494251251,
|
|
"sampling/sampling_logp_difference/max": 0.2702323645353317,
|
|
"sampling/sampling_logp_difference/mean": 0.02107505053281784,
|
|
"step": 9080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 332.2,
|
|
"completions/max_terminated_length": 332.2,
|
|
"completions/mean_length": 223.5125,
|
|
"completions/mean_terminated_length": 223.5125,
|
|
"completions/min_length": 158.8,
|
|
"completions/min_terminated_length": 158.8,
|
|
"entropy": 1.1369658015668391,
|
|
"epoch": 0.8357083754711777,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.20092476904392242,
|
|
"kl": 0.06279710702365264,
|
|
"learning_rate": 4.31942870852894e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 39537977.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.1632926881313324,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.1632926881313324,
|
|
"sampling/importance_sampling_ratio/max": 1.324883759021759,
|
|
"sampling/importance_sampling_ratio/mean": 1.000228124856949,
|
|
"sampling/importance_sampling_ratio/min": 0.7855935573577881,
|
|
"sampling/sampling_logp_difference/max": 0.29696204662323,
|
|
"sampling/sampling_logp_difference/mean": 0.02183693237602711,
|
|
"step": 9090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.0,
|
|
"completions/max_terminated_length": 307.0,
|
|
"completions/mean_length": 219.525,
|
|
"completions/mean_terminated_length": 219.525,
|
|
"completions/min_length": 145.7,
|
|
"completions/min_terminated_length": 145.7,
|
|
"entropy": 1.092097295075655,
|
|
"epoch": 0.836627746621311,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.13455146551132202,
|
|
"kl": 0.05791983523522504,
|
|
"learning_rate": 4.31769016508031e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 39583139.0,
|
|
"reward": 0.04375,
|
|
"reward_std": 0.328694099187851,
|
|
"rewards/reward_fn/mean": 0.04375,
|
|
"rewards/reward_fn/std": 0.32869410514831543,
|
|
"sampling/importance_sampling_ratio/max": 1.2635885000228881,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994966864585877,
|
|
"sampling/importance_sampling_ratio/min": 0.793554937839508,
|
|
"sampling/sampling_logp_difference/max": 0.2487446904182434,
|
|
"sampling/sampling_logp_difference/mean": 0.021288000233471393,
|
|
"step": 9100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.6,
|
|
"completions/max_terminated_length": 302.6,
|
|
"completions/mean_length": 203.5625,
|
|
"completions/mean_terminated_length": 203.5625,
|
|
"completions/min_length": 117.1,
|
|
"completions/min_terminated_length": 117.1,
|
|
"entropy": 0.9976728424429894,
|
|
"epoch": 0.8375471177714443,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.753015995025635,
|
|
"kl": 0.041356326849199834,
|
|
"learning_rate": 4.315949754718174e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 39625208.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.25586686432361605,
|
|
"sampling/importance_sampling_ratio/max": 1.2560867667198181,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998450815677643,
|
|
"sampling/importance_sampling_ratio/min": 0.8101845502853393,
|
|
"sampling/sampling_logp_difference/max": 0.238252592086792,
|
|
"sampling/sampling_logp_difference/mean": 0.020680646784603594,
|
|
"step": 9110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 254.4,
|
|
"completions/max_terminated_length": 254.4,
|
|
"completions/mean_length": 190.7,
|
|
"completions/mean_terminated_length": 190.7,
|
|
"completions/min_length": 132.5,
|
|
"completions/min_terminated_length": 132.5,
|
|
"entropy": 0.9843381345272064,
|
|
"epoch": 0.8384664889215776,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.550142765045166,
|
|
"kl": 0.054185232234885916,
|
|
"learning_rate": 4.3142074792300735e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 39665472.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.28629584312438966,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.2862958490848541,
|
|
"sampling/importance_sampling_ratio/max": 1.3019490242004395,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998579800128937,
|
|
"sampling/importance_sampling_ratio/min": 0.7970300376415252,
|
|
"sampling/sampling_logp_difference/max": 0.2818925380706787,
|
|
"sampling/sampling_logp_difference/mean": 0.020097876340150832,
|
|
"step": 9120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 317.2,
|
|
"completions/max_terminated_length": 317.2,
|
|
"completions/mean_length": 223.1875,
|
|
"completions/mean_terminated_length": 223.1875,
|
|
"completions/min_length": 137.5,
|
|
"completions/min_terminated_length": 137.5,
|
|
"entropy": 1.09163998067379,
|
|
"epoch": 0.8393858600717109,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1480615884065628,
|
|
"kl": 0.04418914269772358,
|
|
"learning_rate": 4.312463340405466e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 39709687.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.32178503572940825,
|
|
"rewards/reward_fn/mean": 0.125,
|
|
"rewards/reward_fn/std": 0.3217850476503372,
|
|
"sampling/importance_sampling_ratio/max": 1.2690747380256653,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998252153396606,
|
|
"sampling/importance_sampling_ratio/min": 0.7846175730228424,
|
|
"sampling/sampling_logp_difference/max": 0.25896793603897095,
|
|
"sampling/sampling_logp_difference/mean": 0.021135289035737515,
|
|
"step": 9130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.1,
|
|
"completions/max_terminated_length": 285.1,
|
|
"completions/mean_length": 203.45,
|
|
"completions/mean_terminated_length": 203.45,
|
|
"completions/min_length": 137.9,
|
|
"completions/min_terminated_length": 137.9,
|
|
"entropy": 1.077194471657276,
|
|
"epoch": 0.8403052312218443,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.326993465423584,
|
|
"kl": 0.046213487058412286,
|
|
"learning_rate": 4.310717340035721e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 39751795.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.2411646068096161,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.2411646068096161,
|
|
"sampling/importance_sampling_ratio/max": 1.2543643355369567,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994263887405396,
|
|
"sampling/importance_sampling_ratio/min": 0.7826475501060486,
|
|
"sampling/sampling_logp_difference/max": 0.25688384771347045,
|
|
"sampling/sampling_logp_difference/mean": 0.02109307814389467,
|
|
"step": 9140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 281.8,
|
|
"completions/max_terminated_length": 281.8,
|
|
"completions/mean_length": 200.55,
|
|
"completions/mean_terminated_length": 200.55,
|
|
"completions/min_length": 133.2,
|
|
"completions/min_terminated_length": 133.2,
|
|
"entropy": 1.1286091461777688,
|
|
"epoch": 0.8412246023719776,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.547336578369141,
|
|
"kl": 0.05496117583825253,
|
|
"learning_rate": 4.308969479914121e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 39795559.0,
|
|
"reward": -0.125,
|
|
"reward_std": 0.2411646068096161,
|
|
"rewards/reward_fn/mean": -0.125,
|
|
"rewards/reward_fn/std": 0.2411646068096161,
|
|
"sampling/importance_sampling_ratio/max": 1.294877290725708,
|
|
"sampling/importance_sampling_ratio/mean": 0.999787712097168,
|
|
"sampling/importance_sampling_ratio/min": 0.7885303974151612,
|
|
"sampling/sampling_logp_difference/max": 0.2806500673294067,
|
|
"sampling/sampling_logp_difference/mean": 0.022265691496431827,
|
|
"step": 9150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.9,
|
|
"completions/max_terminated_length": 312.9,
|
|
"completions/mean_length": 217.475,
|
|
"completions/mean_terminated_length": 217.475,
|
|
"completions/min_length": 129.3,
|
|
"completions/min_terminated_length": 129.3,
|
|
"entropy": 1.0664697647094727,
|
|
"epoch": 0.8421439735221109,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11810187995433807,
|
|
"kl": 0.050366181501885876,
|
|
"learning_rate": 4.307219761835859e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 39843333.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.27019718289375305,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.27019718289375305,
|
|
"sampling/importance_sampling_ratio/max": 1.3031633853912354,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995134472846985,
|
|
"sampling/importance_sampling_ratio/min": 0.7720752239227295,
|
|
"sampling/sampling_logp_difference/max": 0.2952617645263672,
|
|
"sampling/sampling_logp_difference/mean": 0.021450130641460417,
|
|
"step": 9160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 340.2,
|
|
"completions/max_terminated_length": 340.2,
|
|
"completions/mean_length": 222.3125,
|
|
"completions/mean_terminated_length": 222.3125,
|
|
"completions/min_length": 142.7,
|
|
"completions/min_terminated_length": 142.7,
|
|
"entropy": 1.0670593559741974,
|
|
"epoch": 0.8430633446722442,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.911360263824463,
|
|
"kl": 0.055246830178657544,
|
|
"learning_rate": 4.3054681875980345e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 39888246.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.41940403282642363,
|
|
"rewards/reward_fn/mean": 0.2125,
|
|
"rewards/reward_fn/std": 0.41940403282642363,
|
|
"sampling/importance_sampling_ratio/max": 1.2755841493606568,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996138870716095,
|
|
"sampling/importance_sampling_ratio/min": 0.7634301841259002,
|
|
"sampling/sampling_logp_difference/max": 0.2908616304397583,
|
|
"sampling/sampling_logp_difference/mean": 0.021037018671631814,
|
|
"step": 9170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.2,
|
|
"completions/max_terminated_length": 277.2,
|
|
"completions/mean_length": 197.7,
|
|
"completions/mean_terminated_length": 197.7,
|
|
"completions/min_length": 132.9,
|
|
"completions/min_terminated_length": 132.9,
|
|
"entropy": 1.0390971057116984,
|
|
"epoch": 0.8439827158223775,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.16747625172138214,
|
|
"kl": 0.038900166226085275,
|
|
"learning_rate": 4.303714758999655e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 39930822.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.25670533180236815,
|
|
"rewards/reward_fn/mean": 0.4,
|
|
"rewards/reward_fn/std": 0.2567053437232971,
|
|
"sampling/importance_sampling_ratio/max": 1.2425483822822572,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001917123794555,
|
|
"sampling/importance_sampling_ratio/min": 0.7860532343387604,
|
|
"sampling/sampling_logp_difference/max": 0.24847247898578645,
|
|
"sampling/sampling_logp_difference/mean": 0.02065215855836868,
|
|
"step": 9180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.5,
|
|
"completions/max_terminated_length": 261.5,
|
|
"completions/mean_length": 183.425,
|
|
"completions/mean_terminated_length": 183.425,
|
|
"completions/min_length": 111.9,
|
|
"completions/min_terminated_length": 111.9,
|
|
"entropy": 1.0045614168047905,
|
|
"epoch": 0.8449020869725108,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.19063769280910492,
|
|
"kl": 0.039167750254273416,
|
|
"learning_rate": 4.301959477841634e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 39971376.0,
|
|
"reward": 0.5125,
|
|
"reward_std": 0.2722664296627045,
|
|
"rewards/reward_fn/mean": 0.5125,
|
|
"rewards/reward_fn/std": 0.2722664475440979,
|
|
"sampling/importance_sampling_ratio/max": 1.2711398124694824,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999018847942353,
|
|
"sampling/importance_sampling_ratio/min": 0.7953747391700745,
|
|
"sampling/sampling_logp_difference/max": 0.25212845802307127,
|
|
"sampling/sampling_logp_difference/mean": 0.02159304954111576,
|
|
"step": 9190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.1,
|
|
"completions/max_terminated_length": 285.1,
|
|
"completions/mean_length": 196.2375,
|
|
"completions/mean_terminated_length": 196.2375,
|
|
"completions/min_length": 115.0,
|
|
"completions/min_terminated_length": 115.0,
|
|
"entropy": 1.0011788252741098,
|
|
"epoch": 0.8458214581226441,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.14685741066932678,
|
|
"kl": 0.04386705849319696,
|
|
"learning_rate": 4.300202345926783e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 40014771.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.2903633385896683,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.2903633564710617,
|
|
"sampling/importance_sampling_ratio/max": 1.2652434349060058,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998990893363953,
|
|
"sampling/importance_sampling_ratio/min": 0.7806730091571807,
|
|
"sampling/sampling_logp_difference/max": 0.2740874409675598,
|
|
"sampling/sampling_logp_difference/mean": 0.020218341425061227,
|
|
"step": 9200
|
|
},
|
|
{
|
|
"epoch": 0.8458214581226441,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0017452006980802793,
|
|
"eval_completions/max_length": 280.3438045375218,
|
|
"eval_completions/max_terminated_length": 277.4397905759162,
|
|
"eval_completions/mean_length": 199.8215532286213,
|
|
"eval_completions/mean_terminated_length": 199.32784853098073,
|
|
"eval_completions/min_length": 128.62303664921467,
|
|
"eval_completions/min_terminated_length": 128.62303664921467,
|
|
"eval_entropy": 1.0598399707264092,
|
|
"eval_frac_reward_zero_std": 0.3403141361256545,
|
|
"eval_kl": 0.04085794858793756,
|
|
"eval_loss": 0.002103661885485053,
|
|
"eval_num_tokens": 40014771.0,
|
|
"eval_reward": 0.07907940663176265,
|
|
"eval_reward_std": 0.33248850828065923,
|
|
"eval_rewards/reward_fn/mean": 0.07907940663176265,
|
|
"eval_rewards/reward_fn/std": 0.3324885173305791,
|
|
"eval_runtime": 1648.4789,
|
|
"eval_samples_per_second": 0.348,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2619268696553627,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999836871553258,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7830266549949246,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2691153499572481,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.020553737670755304,
|
|
"eval_steps_per_second": 0.044,
|
|
"step": 9200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.3,
|
|
"completions/max_terminated_length": 306.3,
|
|
"completions/mean_length": 207.7625,
|
|
"completions/mean_terminated_length": 207.7625,
|
|
"completions/min_length": 134.3,
|
|
"completions/min_terminated_length": 134.3,
|
|
"entropy": 1.0039683520793914,
|
|
"epoch": 0.8467408292727774,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.9489359855651855,
|
|
"kl": 0.04009920536191203,
|
|
"learning_rate": 4.29844336505982e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 40058520.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.37287638187408445,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.37287638187408445,
|
|
"sampling/importance_sampling_ratio/max": 1.2430763840675354,
|
|
"sampling/importance_sampling_ratio/mean": 1.000257235765457,
|
|
"sampling/importance_sampling_ratio/min": 0.7770889043807984,
|
|
"sampling/sampling_logp_difference/max": 0.27759191393852234,
|
|
"sampling/sampling_logp_difference/mean": 0.01987345777451992,
|
|
"step": 9210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.0,
|
|
"completions/max_terminated_length": 298.0,
|
|
"completions/mean_length": 197.8875,
|
|
"completions/mean_terminated_length": 197.8875,
|
|
"completions/min_length": 125.6,
|
|
"completions/min_terminated_length": 125.6,
|
|
"entropy": 1.1288753256201745,
|
|
"epoch": 0.8476602004229107,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.102019309997559,
|
|
"kl": 0.04638625959632918,
|
|
"learning_rate": 4.296682537047357e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 40102391.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.31855743527412417,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.3185574531555176,
|
|
"sampling/importance_sampling_ratio/max": 1.291967785358429,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998250126838684,
|
|
"sampling/importance_sampling_ratio/min": 0.7486029088497161,
|
|
"sampling/sampling_logp_difference/max": 0.3039748251438141,
|
|
"sampling/sampling_logp_difference/mean": 0.022176310047507287,
|
|
"step": 9220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.5,
|
|
"completions/max_terminated_length": 299.5,
|
|
"completions/mean_length": 217.95,
|
|
"completions/mean_terminated_length": 217.95,
|
|
"completions/min_length": 146.8,
|
|
"completions/min_terminated_length": 146.8,
|
|
"entropy": 1.076299573481083,
|
|
"epoch": 0.8485795715730441,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.501277446746826,
|
|
"kl": 0.041976634215097876,
|
|
"learning_rate": 4.294919863697908e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 40146483.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.36099914014339446,
|
|
"rewards/reward_fn/mean": 0.4625,
|
|
"rewards/reward_fn/std": 0.3609991520643234,
|
|
"sampling/importance_sampling_ratio/max": 1.2423346042633057,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995251178741456,
|
|
"sampling/importance_sampling_ratio/min": 0.7751516163349151,
|
|
"sampling/sampling_logp_difference/max": 0.2673645973205566,
|
|
"sampling/sampling_logp_difference/mean": 0.02062288634479046,
|
|
"step": 9230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.6,
|
|
"completions/max_terminated_length": 280.6,
|
|
"completions/mean_length": 204.4875,
|
|
"completions/mean_terminated_length": 204.4875,
|
|
"completions/min_length": 147.6,
|
|
"completions/min_terminated_length": 147.6,
|
|
"entropy": 1.1663407430052757,
|
|
"epoch": 0.8494989427231774,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.10546322911977768,
|
|
"kl": 0.03811493905377574,
|
|
"learning_rate": 4.293155346821877e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 40187570.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.501947420835495,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.501947432756424,
|
|
"sampling/importance_sampling_ratio/max": 1.2799391627311707,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999857306480407,
|
|
"sampling/importance_sampling_ratio/min": 0.7642562031745911,
|
|
"sampling/sampling_logp_difference/max": 0.297974967956543,
|
|
"sampling/sampling_logp_difference/mean": 0.022754647955298423,
|
|
"step": 9240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.9,
|
|
"completions/max_terminated_length": 298.9,
|
|
"completions/mean_length": 216.875,
|
|
"completions/mean_terminated_length": 216.875,
|
|
"completions/min_length": 135.0,
|
|
"completions/min_terminated_length": 135.0,
|
|
"entropy": 1.0723770407959818,
|
|
"epoch": 0.8504183138733107,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.826480388641357,
|
|
"kl": 0.04773159505566582,
|
|
"learning_rate": 4.291388988231567e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 40232080.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.338620987534523,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.338620999455452,
|
|
"sampling/importance_sampling_ratio/max": 1.261952781677246,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998500108718872,
|
|
"sampling/importance_sampling_ratio/min": 0.7857359766960144,
|
|
"sampling/sampling_logp_difference/max": 0.25415499210357667,
|
|
"sampling/sampling_logp_difference/mean": 0.02116878693923354,
|
|
"step": 9250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 285.5,
|
|
"completions/max_terminated_length": 280.5,
|
|
"completions/mean_length": 212.0125,
|
|
"completions/mean_terminated_length": 205.42749938964843,
|
|
"completions/min_length": 150.5,
|
|
"completions/min_terminated_length": 150.5,
|
|
"entropy": 1.0163930386304856,
|
|
"epoch": 0.851337685023444,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 7.354737281799316,
|
|
"kl": 0.03842501549515873,
|
|
"learning_rate": 4.289620789741169e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 40275121.0,
|
|
"reward": 0.04375,
|
|
"reward_std": 0.26761444509029386,
|
|
"rewards/reward_fn/mean": 0.04375,
|
|
"rewards/reward_fn/std": 0.26761446297168734,
|
|
"sampling/importance_sampling_ratio/max": 1.2281743884086609,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004379153251648,
|
|
"sampling/importance_sampling_ratio/min": 0.7780608057975769,
|
|
"sampling/sampling_logp_difference/max": 0.25594727993011473,
|
|
"sampling/sampling_logp_difference/mean": 0.020437494665384293,
|
|
"step": 9260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.4,
|
|
"completions/max_terminated_length": 286.4,
|
|
"completions/mean_length": 207.85,
|
|
"completions/mean_terminated_length": 207.85,
|
|
"completions/min_length": 135.4,
|
|
"completions/min_terminated_length": 135.4,
|
|
"entropy": 1.0362411573529244,
|
|
"epoch": 0.8522570561735773,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.695846080780029,
|
|
"kl": 0.03845738079980947,
|
|
"learning_rate": 4.2878507531667635e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 40316525.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.4025459736585617,
|
|
"rewards/reward_fn/mean": 0.21875,
|
|
"rewards/reward_fn/std": 0.40254599750041964,
|
|
"sampling/importance_sampling_ratio/max": 1.347732961177826,
|
|
"sampling/importance_sampling_ratio/mean": 1.0007512390613555,
|
|
"sampling/importance_sampling_ratio/min": 0.8026901960372925,
|
|
"sampling/sampling_logp_difference/max": 0.2978661060333252,
|
|
"sampling/sampling_logp_difference/mean": 0.020542396046221257,
|
|
"step": 9270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.2,
|
|
"completions/max_terminated_length": 274.2,
|
|
"completions/mean_length": 203.0,
|
|
"completions/mean_terminated_length": 203.0,
|
|
"completions/min_length": 130.6,
|
|
"completions/min_terminated_length": 130.6,
|
|
"entropy": 1.0238441742956639,
|
|
"epoch": 0.8531764273237106,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.857336044311523,
|
|
"kl": 0.04517930138972588,
|
|
"learning_rate": 4.286078880326321e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 40364389.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.29780173897743223,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.29780174493789674,
|
|
"sampling/importance_sampling_ratio/max": 1.275265920162201,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000287234783172,
|
|
"sampling/importance_sampling_ratio/min": 0.7684423327445984,
|
|
"sampling/sampling_logp_difference/max": 0.28910396099090574,
|
|
"sampling/sampling_logp_difference/mean": 0.020778720453381538,
|
|
"step": 9280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.3,
|
|
"completions/max_terminated_length": 269.3,
|
|
"completions/mean_length": 200.55,
|
|
"completions/mean_terminated_length": 200.55,
|
|
"completions/min_length": 144.9,
|
|
"completions/min_terminated_length": 144.9,
|
|
"entropy": 1.1211679950356483,
|
|
"epoch": 0.8540957984738439,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.11894954741001129,
|
|
"kl": 0.04322960283025168,
|
|
"learning_rate": 4.284305173039697e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 40407969.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.3685124158859253,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.36851242184638977,
|
|
"sampling/importance_sampling_ratio/max": 1.3969801306724547,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002871692180633,
|
|
"sampling/importance_sampling_ratio/min": 0.7616239607334137,
|
|
"sampling/sampling_logp_difference/max": 0.4036160588264465,
|
|
"sampling/sampling_logp_difference/mean": 0.022723359428346156,
|
|
"step": 9290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 246.9,
|
|
"completions/max_terminated_length": 246.9,
|
|
"completions/mean_length": 180.3,
|
|
"completions/mean_terminated_length": 180.3,
|
|
"completions/min_length": 123.0,
|
|
"completions/min_terminated_length": 123.0,
|
|
"entropy": 1.0072774156928062,
|
|
"epoch": 0.8550151696239772,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.7574663162231445,
|
|
"kl": 0.05088316059554927,
|
|
"learning_rate": 4.28252963312863e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 40448225.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.3564485847949982,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.3564485967159271,
|
|
"sampling/importance_sampling_ratio/max": 1.2754207134246827,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002938032150268,
|
|
"sampling/importance_sampling_ratio/min": 0.7954947233200074,
|
|
"sampling/sampling_logp_difference/max": 0.255824875831604,
|
|
"sampling/sampling_logp_difference/mean": 0.020475606620311736,
|
|
"step": 9300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 302.6,
|
|
"completions/max_terminated_length": 292.3,
|
|
"completions/mean_length": 207.1625,
|
|
"completions/mean_terminated_length": 204.79107360839845,
|
|
"completions/min_length": 143.6,
|
|
"completions/min_terminated_length": 143.6,
|
|
"entropy": 1.0452521368861198,
|
|
"epoch": 0.8559345407741105,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.0919904708862305,
|
|
"kl": 0.028760029276600107,
|
|
"learning_rate": 4.280752262416743e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 40488478.0,
|
|
"reward": 0.35625,
|
|
"reward_std": 0.41326538026332854,
|
|
"rewards/reward_fn/mean": 0.35625,
|
|
"rewards/reward_fn/std": 0.41326538026332854,
|
|
"sampling/importance_sampling_ratio/max": 1.253693902492523,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000602841377257,
|
|
"sampling/importance_sampling_ratio/min": 0.7971062898635864,
|
|
"sampling/sampling_logp_difference/max": 0.2408949613571167,
|
|
"sampling/sampling_logp_difference/mean": 0.02095355149358511,
|
|
"step": 9310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 344.4,
|
|
"completions/max_terminated_length": 342.5,
|
|
"completions/mean_length": 257.3125,
|
|
"completions/mean_terminated_length": 255.1875,
|
|
"completions/min_length": 190.1,
|
|
"completions/min_terminated_length": 190.1,
|
|
"entropy": 1.0408134903758763,
|
|
"epoch": 0.8568539119242439,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.6079256534576416,
|
|
"kl": 0.04141123883891851,
|
|
"learning_rate": 4.2789730627295375e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 40535351.0,
|
|
"reward": -0.21875,
|
|
"reward_std": 0.49128166437149046,
|
|
"rewards/reward_fn/mean": -0.21875,
|
|
"rewards/reward_fn/std": 0.4912816762924194,
|
|
"sampling/importance_sampling_ratio/max": 1.244085967540741,
|
|
"sampling/importance_sampling_ratio/mean": 0.999877679347992,
|
|
"sampling/importance_sampling_ratio/min": 0.767325896024704,
|
|
"sampling/sampling_logp_difference/max": 0.27972211837768557,
|
|
"sampling/sampling_logp_difference/mean": 0.019891233555972575,
|
|
"step": 9320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.0,
|
|
"completions/max_terminated_length": 299.0,
|
|
"completions/mean_length": 209.475,
|
|
"completions/mean_terminated_length": 209.475,
|
|
"completions/min_length": 134.8,
|
|
"completions/min_terminated_length": 134.8,
|
|
"entropy": 1.053185585141182,
|
|
"epoch": 0.8577732830743772,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 7.106479167938232,
|
|
"kl": 0.0601667522219941,
|
|
"learning_rate": 4.277192035894394e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 40582397.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.2276727557182312,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.22767276763916017,
|
|
"sampling/importance_sampling_ratio/max": 1.2634025931358337,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998867154121399,
|
|
"sampling/importance_sampling_ratio/min": 0.779437643289566,
|
|
"sampling/sampling_logp_difference/max": 0.2719349145889282,
|
|
"sampling/sampling_logp_difference/mean": 0.021227684058248996,
|
|
"step": 9330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.0,
|
|
"completions/max_terminated_length": 306.0,
|
|
"completions/mean_length": 212.3375,
|
|
"completions/mean_terminated_length": 212.3375,
|
|
"completions/min_length": 139.1,
|
|
"completions/min_terminated_length": 139.1,
|
|
"entropy": 1.0828462682664395,
|
|
"epoch": 0.8586926542245105,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 7.119572639465332,
|
|
"kl": 0.05455377746839076,
|
|
"learning_rate": 4.275409183740571e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 40626552.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.07071067690849304,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.07071067690849304,
|
|
"sampling/importance_sampling_ratio/max": 1.2365211606025697,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997987985610962,
|
|
"sampling/importance_sampling_ratio/min": 0.7920197784900666,
|
|
"sampling/sampling_logp_difference/max": 0.2397668719291687,
|
|
"sampling/sampling_logp_difference/mean": 0.021641866490244865,
|
|
"step": 9340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.1,
|
|
"completions/max_terminated_length": 313.1,
|
|
"completions/mean_length": 210.9125,
|
|
"completions/mean_terminated_length": 210.9125,
|
|
"completions/min_length": 119.9,
|
|
"completions/min_terminated_length": 119.9,
|
|
"entropy": 1.0368298828601836,
|
|
"epoch": 0.8596120253746438,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.060151100158691,
|
|
"kl": 0.05147368920734152,
|
|
"learning_rate": 4.2736245080992e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 40671137.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.2504029482603073,
|
|
"sampling/importance_sampling_ratio/max": 1.2670783519744873,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001860976219177,
|
|
"sampling/importance_sampling_ratio/min": 0.7762269496917724,
|
|
"sampling/sampling_logp_difference/max": 0.27801127433776857,
|
|
"sampling/sampling_logp_difference/mean": 0.021501859277486803,
|
|
"step": 9350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.7,
|
|
"completions/max_terminated_length": 313.7,
|
|
"completions/mean_length": 210.0375,
|
|
"completions/mean_terminated_length": 210.0375,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 1.0472055025398732,
|
|
"epoch": 0.8605313965247771,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.16727247834205627,
|
|
"kl": 0.050492165464675054,
|
|
"learning_rate": 4.271838010803285e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 40712804.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.2205115258693695,
|
|
"sampling/importance_sampling_ratio/max": 1.27891263961792,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001486480236053,
|
|
"sampling/importance_sampling_ratio/min": 0.7984284102916718,
|
|
"sampling/sampling_logp_difference/max": 0.26039583683013917,
|
|
"sampling/sampling_logp_difference/mean": 0.02063222713768482,
|
|
"step": 9360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 260.7,
|
|
"completions/max_terminated_length": 260.7,
|
|
"completions/mean_length": 184.6,
|
|
"completions/mean_terminated_length": 184.6,
|
|
"completions/min_length": 123.0,
|
|
"completions/min_terminated_length": 123.0,
|
|
"entropy": 1.0257703594863414,
|
|
"epoch": 0.8614507676749104,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.872483253479004,
|
|
"kl": 0.041812674043467266,
|
|
"learning_rate": 4.270049693687703e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 40752196.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.3038551896810532,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.30385519564151764,
|
|
"sampling/importance_sampling_ratio/max": 1.2589488625526428,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001261413097382,
|
|
"sampling/importance_sampling_ratio/min": 0.786538028717041,
|
|
"sampling/sampling_logp_difference/max": 0.2517038404941559,
|
|
"sampling/sampling_logp_difference/mean": 0.020944275427609682,
|
|
"step": 9370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 321.8,
|
|
"completions/max_terminated_length": 321.1,
|
|
"completions/mean_length": 220.925,
|
|
"completions/mean_terminated_length": 218.57142944335936,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 1.1077851600944997,
|
|
"epoch": 0.8623701388250437,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.332583427429199,
|
|
"kl": 0.035996814008103686,
|
|
"learning_rate": 4.268259558589199e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 40794974.0,
|
|
"reward": 0.00625,
|
|
"reward_std": 0.4347910940647125,
|
|
"rewards/reward_fn/mean": 0.00625,
|
|
"rewards/reward_fn/std": 0.434791100025177,
|
|
"sampling/importance_sampling_ratio/max": 1.2473416090011598,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004087567329407,
|
|
"sampling/importance_sampling_ratio/min": 0.762635612487793,
|
|
"sampling/sampling_logp_difference/max": 0.2881794452667236,
|
|
"sampling/sampling_logp_difference/mean": 0.021480763517320155,
|
|
"step": 9380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.9,
|
|
"completions/max_terminated_length": 293.9,
|
|
"completions/mean_length": 210.7625,
|
|
"completions/mean_terminated_length": 210.7625,
|
|
"completions/min_length": 127.9,
|
|
"completions/min_terminated_length": 127.9,
|
|
"entropy": 0.9693314112722874,
|
|
"epoch": 0.863289509975177,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 8.771162986755371,
|
|
"kl": 0.04351616412168369,
|
|
"learning_rate": 4.266467607346385e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 40834755.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.261338609457016,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.2613386154174805,
|
|
"sampling/importance_sampling_ratio/max": 1.338568377494812,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003405570983888,
|
|
"sampling/importance_sampling_ratio/min": 0.7924472033977509,
|
|
"sampling/sampling_logp_difference/max": 0.29763525128364565,
|
|
"sampling/sampling_logp_difference/mean": 0.019548206962645052,
|
|
"step": 9390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.0,
|
|
"completions/max_terminated_length": 326.0,
|
|
"completions/mean_length": 225.0875,
|
|
"completions/mean_terminated_length": 225.0875,
|
|
"completions/min_length": 141.7,
|
|
"completions/min_terminated_length": 141.7,
|
|
"entropy": 1.1542256452143191,
|
|
"epoch": 0.8642088811253102,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.076078414916992,
|
|
"kl": 0.03984848668333143,
|
|
"learning_rate": 4.264673841799738e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 40879154.0,
|
|
"reward": 0.33125,
|
|
"reward_std": 0.36538374722003936,
|
|
"rewards/reward_fn/mean": 0.33125,
|
|
"rewards/reward_fn/std": 0.36538375318050387,
|
|
"sampling/importance_sampling_ratio/max": 1.29695383310318,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999223828315735,
|
|
"sampling/importance_sampling_ratio/min": 0.7794753611087799,
|
|
"sampling/sampling_logp_difference/max": 0.28198263645172117,
|
|
"sampling/sampling_logp_difference/mean": 0.0225184241309762,
|
|
"step": 9400
|
|
},
|
|
{
|
|
"epoch": 0.8642088811253102,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0008726003490401396,
|
|
"eval_completions/max_length": 294.6282722513089,
|
|
"eval_completions/max_terminated_length": 293.7434554973822,
|
|
"eval_completions/mean_length": 211.76941535776615,
|
|
"eval_completions/mean_terminated_length": 211.53758415501778,
|
|
"eval_completions/min_length": 141.4240837696335,
|
|
"eval_completions/min_terminated_length": 141.4240837696335,
|
|
"eval_entropy": 1.1172681619359561,
|
|
"eval_frac_reward_zero_std": 0.35951134380453753,
|
|
"eval_kl": 0.04241976077240903,
|
|
"eval_loss": 0.0022181645035743713,
|
|
"eval_num_tokens": 40879154.0,
|
|
"eval_reward": 0.07831588132635253,
|
|
"eval_reward_std": 0.31453906210305177,
|
|
"eval_rewards/reward_fn/mean": 0.07831588132635253,
|
|
"eval_rewards/reward_fn/std": 0.31453907240123646,
|
|
"eval_runtime": 1721.9439,
|
|
"eval_samples_per_second": 0.333,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2717437896012844,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000231140155027,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.774518882468941,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2820114487128732,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02131458327830826,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 9400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 278.4,
|
|
"completions/max_terminated_length": 278.4,
|
|
"completions/mean_length": 206.6625,
|
|
"completions/mean_terminated_length": 206.6625,
|
|
"completions/min_length": 139.0,
|
|
"completions/min_terminated_length": 139.0,
|
|
"entropy": 1.0495636306703091,
|
|
"epoch": 0.8651282522754435,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.368381977081299,
|
|
"kl": 0.032928543823072684,
|
|
"learning_rate": 4.2628782637916e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 40921431.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.23691109120845794,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.23691110908985138,
|
|
"sampling/importance_sampling_ratio/max": 1.303182554244995,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000178515911102,
|
|
"sampling/importance_sampling_ratio/min": 0.7715028166770935,
|
|
"sampling/sampling_logp_difference/max": 0.29765647649765015,
|
|
"sampling/sampling_logp_difference/mean": 0.021498052589595317,
|
|
"step": 9410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.5,
|
|
"completions/max_terminated_length": 297.5,
|
|
"completions/mean_length": 202.375,
|
|
"completions/mean_terminated_length": 202.375,
|
|
"completions/min_length": 140.8,
|
|
"completions/min_terminated_length": 140.8,
|
|
"entropy": 1.1371506549417973,
|
|
"epoch": 0.866047623425577,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.960000038146973,
|
|
"kl": 0.058892602392006665,
|
|
"learning_rate": 4.261080875166173e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 40964781.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.3086789697408676,
|
|
"rewards/reward_fn/mean": 0.05,
|
|
"rewards/reward_fn/std": 0.3086789816617966,
|
|
"sampling/importance_sampling_ratio/max": 1.2429131627082826,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996340036392212,
|
|
"sampling/importance_sampling_ratio/min": 0.73636794090271,
|
|
"sampling/sampling_logp_difference/max": 0.31735005378723147,
|
|
"sampling/sampling_logp_difference/mean": 0.02245978843420744,
|
|
"step": 9420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.8,
|
|
"completions/max_terminated_length": 299.8,
|
|
"completions/mean_length": 212.7875,
|
|
"completions/mean_terminated_length": 212.7875,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 1.0053416728973388,
|
|
"epoch": 0.8669669945757102,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.631198406219482,
|
|
"kl": 0.042906089848838744,
|
|
"learning_rate": 4.25928167776952e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 41015988.0,
|
|
"reward": -0.1875,
|
|
"reward_std": 0.4560851901769638,
|
|
"rewards/reward_fn/mean": -0.1875,
|
|
"rewards/reward_fn/std": 0.4560852020978928,
|
|
"sampling/importance_sampling_ratio/max": 1.2623692631721497,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996741950511933,
|
|
"sampling/importance_sampling_ratio/min": 0.775529658794403,
|
|
"sampling/sampling_logp_difference/max": 0.26469323635101316,
|
|
"sampling/sampling_logp_difference/mean": 0.02053749095648527,
|
|
"step": 9430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.5,
|
|
"completions/max_terminated_length": 290.5,
|
|
"completions/mean_length": 200.8375,
|
|
"completions/mean_terminated_length": 200.8375,
|
|
"completions/min_length": 140.0,
|
|
"completions/min_terminated_length": 140.0,
|
|
"entropy": 1.0861917831003667,
|
|
"epoch": 0.8678863657258435,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12043377757072449,
|
|
"kl": 0.060240122966934,
|
|
"learning_rate": 4.2574806734495585e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 41059047.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.25586686432361605,
|
|
"sampling/importance_sampling_ratio/max": 1.3006654381752014,
|
|
"sampling/importance_sampling_ratio/mean": 1.000587022304535,
|
|
"sampling/importance_sampling_ratio/min": 0.7896238088607788,
|
|
"sampling/sampling_logp_difference/max": 0.2809868693351746,
|
|
"sampling/sampling_logp_difference/mean": 0.021761747635900976,
|
|
"step": 9440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.8,
|
|
"completions/max_terminated_length": 301.8,
|
|
"completions/mean_length": 215.0625,
|
|
"completions/mean_terminated_length": 215.0625,
|
|
"completions/min_length": 144.7,
|
|
"completions/min_terminated_length": 144.7,
|
|
"entropy": 1.1245507352054118,
|
|
"epoch": 0.8688057368759768,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.208584308624268,
|
|
"kl": 0.03909367670421489,
|
|
"learning_rate": 4.2556778640560684e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 41105612.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.3389096111059189,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.33890961706638334,
|
|
"sampling/importance_sampling_ratio/max": 1.3357073187828064,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006796002388,
|
|
"sampling/importance_sampling_ratio/min": 0.7836547791957855,
|
|
"sampling/sampling_logp_difference/max": 0.3044546961784363,
|
|
"sampling/sampling_logp_difference/mean": 0.021762259677052496,
|
|
"step": 9450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.6,
|
|
"completions/max_terminated_length": 303.6,
|
|
"completions/mean_length": 219.025,
|
|
"completions/mean_terminated_length": 219.025,
|
|
"completions/min_length": 139.7,
|
|
"completions/min_terminated_length": 139.7,
|
|
"entropy": 1.093241859972477,
|
|
"epoch": 0.8697251080261101,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.2245897501707077,
|
|
"kl": 0.03936568092321977,
|
|
"learning_rate": 4.253873251440675e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 41149750.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.33463136851787567,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.3346313744783401,
|
|
"sampling/importance_sampling_ratio/max": 1.3437366247177125,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005652964115144,
|
|
"sampling/importance_sampling_ratio/min": 0.7690230071544647,
|
|
"sampling/sampling_logp_difference/max": 0.32501999139785764,
|
|
"sampling/sampling_logp_difference/mean": 0.02135617621243,
|
|
"step": 9460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.5,
|
|
"completions/max_terminated_length": 299.5,
|
|
"completions/mean_length": 218.7875,
|
|
"completions/mean_terminated_length": 218.7875,
|
|
"completions/min_length": 148.8,
|
|
"completions/min_terminated_length": 148.8,
|
|
"entropy": 1.0457527928054333,
|
|
"epoch": 0.8706444791762434,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11355587840080261,
|
|
"kl": 0.050437202595639975,
|
|
"learning_rate": 4.252066837456863e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 41196141.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.25756419003009795,
|
|
"sampling/importance_sampling_ratio/max": 1.2807958245277404,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000260055065155,
|
|
"sampling/importance_sampling_ratio/min": 0.7765036642551422,
|
|
"sampling/sampling_logp_difference/max": 0.27603033781051634,
|
|
"sampling/sampling_logp_difference/mean": 0.021162834577262403,
|
|
"step": 9470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.1,
|
|
"completions/max_terminated_length": 282.1,
|
|
"completions/mean_length": 193.7,
|
|
"completions/mean_terminated_length": 193.7,
|
|
"completions/min_length": 131.1,
|
|
"completions/min_terminated_length": 131.1,
|
|
"entropy": 1.0732170723378658,
|
|
"epoch": 0.8715638503263767,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11755069345235825,
|
|
"kl": 0.04790999605320394,
|
|
"learning_rate": 4.250258623959964e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 41236821.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.3255516141653061,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.325551638007164,
|
|
"sampling/importance_sampling_ratio/max": 1.2618190407752992,
|
|
"sampling/importance_sampling_ratio/mean": 0.99973384141922,
|
|
"sampling/importance_sampling_ratio/min": 0.7422408819198608,
|
|
"sampling/sampling_logp_difference/max": 0.3121058762073517,
|
|
"sampling/sampling_logp_difference/mean": 0.0211736012250185,
|
|
"step": 9480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.0,
|
|
"completions/max_terminated_length": 290.0,
|
|
"completions/mean_length": 208.4875,
|
|
"completions/mean_terminated_length": 208.4875,
|
|
"completions/min_length": 139.4,
|
|
"completions/min_terminated_length": 139.4,
|
|
"entropy": 1.064863720536232,
|
|
"epoch": 0.87248322147651,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.2439799308776855,
|
|
"kl": 0.043975573859643194,
|
|
"learning_rate": 4.248448612807158e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 41280268.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.3689270466566086,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.368927064538002,
|
|
"sampling/importance_sampling_ratio/max": 1.262490689754486,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003486037254334,
|
|
"sampling/importance_sampling_ratio/min": 0.7779062211513519,
|
|
"sampling/sampling_logp_difference/max": 0.27479383945465086,
|
|
"sampling/sampling_logp_difference/mean": 0.021718551218509675,
|
|
"step": 9490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 337.0,
|
|
"completions/max_terminated_length": 337.0,
|
|
"completions/mean_length": 239.975,
|
|
"completions/mean_terminated_length": 239.975,
|
|
"completions/min_length": 157.2,
|
|
"completions/min_terminated_length": 157.2,
|
|
"entropy": 1.0880503192543984,
|
|
"epoch": 0.8734025926266433,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.11339802294969559,
|
|
"kl": 0.043944471870781855,
|
|
"learning_rate": 4.246636805857471e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 41329050.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.2328435957431793,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.2328436017036438,
|
|
"sampling/importance_sampling_ratio/max": 1.2938705444335938,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003528118133544,
|
|
"sampling/importance_sampling_ratio/min": 0.7880222916603088,
|
|
"sampling/sampling_logp_difference/max": 0.2763191759586334,
|
|
"sampling/sampling_logp_difference/mean": 0.020726339891552925,
|
|
"step": 9500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 265.4,
|
|
"completions/max_terminated_length": 265.4,
|
|
"completions/mean_length": 200.0875,
|
|
"completions/mean_terminated_length": 200.0875,
|
|
"completions/min_length": 141.3,
|
|
"completions/min_terminated_length": 141.3,
|
|
"entropy": 1.146623032540083,
|
|
"epoch": 0.8743219637767767,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.862491607666016,
|
|
"kl": 0.047342038265196604,
|
|
"learning_rate": 4.244823204971773e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 41371377.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.26680251955986023,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.2668025314807892,
|
|
"sampling/importance_sampling_ratio/max": 1.2572236895561217,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999559581279754,
|
|
"sampling/importance_sampling_ratio/min": 0.7586696684360504,
|
|
"sampling/sampling_logp_difference/max": 0.29272322058677674,
|
|
"sampling/sampling_logp_difference/mean": 0.022369664534926413,
|
|
"step": 9510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.0,
|
|
"completions/max_terminated_length": 304.0,
|
|
"completions/mean_length": 221.125,
|
|
"completions/mean_terminated_length": 221.125,
|
|
"completions/min_length": 151.0,
|
|
"completions/min_terminated_length": 151.0,
|
|
"entropy": 1.0671612076461314,
|
|
"epoch": 0.87524133492691,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.720339298248291,
|
|
"kl": 0.04889992101234384,
|
|
"learning_rate": 4.24300781201278e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 41413307.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.44760257601737974,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.4476025938987732,
|
|
"sampling/importance_sampling_ratio/max": 1.3142322897911072,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997621655464173,
|
|
"sampling/importance_sampling_ratio/min": 0.7700957477092742,
|
|
"sampling/sampling_logp_difference/max": 0.2946740388870239,
|
|
"sampling/sampling_logp_difference/mean": 0.021997046284377576,
|
|
"step": 9520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.3,
|
|
"completions/max_terminated_length": 312.3,
|
|
"completions/mean_length": 229.6125,
|
|
"completions/mean_terminated_length": 229.6125,
|
|
"completions/min_length": 153.4,
|
|
"completions/min_terminated_length": 153.4,
|
|
"entropy": 1.2160911850631237,
|
|
"epoch": 0.8761607060770433,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.15232910215854645,
|
|
"kl": 0.03515291656367481,
|
|
"learning_rate": 4.2411906288450446e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 41461516.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.1632926881313324,
|
|
"rewards/reward_fn/mean": 0.5,
|
|
"rewards/reward_fn/std": 0.1632926881313324,
|
|
"sampling/importance_sampling_ratio/max": 1.2607879400253297,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998975217342376,
|
|
"sampling/importance_sampling_ratio/min": 0.7959343075752259,
|
|
"sampling/sampling_logp_difference/max": 0.2527146220207214,
|
|
"sampling/sampling_logp_difference/mean": 0.02243077214807272,
|
|
"step": 9530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.2,
|
|
"completions/max_terminated_length": 303.2,
|
|
"completions/mean_length": 216.95,
|
|
"completions/mean_terminated_length": 216.95,
|
|
"completions/min_length": 147.5,
|
|
"completions/min_terminated_length": 147.5,
|
|
"entropy": 1.095352490991354,
|
|
"epoch": 0.8770800772271766,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.13643012940883636,
|
|
"kl": 0.05331411722581834,
|
|
"learning_rate": 4.23937165733496e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 41502536.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.3486855089664459,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.34868552088737487,
|
|
"sampling/importance_sampling_ratio/max": 1.4292441010475159,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999718368053436,
|
|
"sampling/importance_sampling_ratio/min": 0.7763973474502563,
|
|
"sampling/sampling_logp_difference/max": 0.4093906044960022,
|
|
"sampling/sampling_logp_difference/mean": 0.021223792061209678,
|
|
"step": 9540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 322.9,
|
|
"completions/max_terminated_length": 322.9,
|
|
"completions/mean_length": 220.5875,
|
|
"completions/mean_terminated_length": 220.5875,
|
|
"completions/min_length": 144.1,
|
|
"completions/min_terminated_length": 144.1,
|
|
"entropy": 1.1236950181424619,
|
|
"epoch": 0.8779994483773099,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12278574705123901,
|
|
"kl": 0.03723166879499331,
|
|
"learning_rate": 4.237550899350755e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 41547423.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.3257186770439148,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.32571869492530825,
|
|
"sampling/importance_sampling_ratio/max": 1.319928765296936,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999402165412903,
|
|
"sampling/importance_sampling_ratio/min": 0.7845882117748261,
|
|
"sampling/sampling_logp_difference/max": 0.2943567276000977,
|
|
"sampling/sampling_logp_difference/mean": 0.022001724503934384,
|
|
"step": 9550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.7,
|
|
"completions/max_terminated_length": 313.7,
|
|
"completions/mean_length": 222.7875,
|
|
"completions/mean_terminated_length": 222.7875,
|
|
"completions/min_length": 155.5,
|
|
"completions/min_terminated_length": 155.5,
|
|
"entropy": 1.0925446793437004,
|
|
"epoch": 0.8789188195274432,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.862927436828613,
|
|
"kl": 0.046284791472135114,
|
|
"learning_rate": 4.235728356762496e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 41590598.0,
|
|
"reward": -0.2625,
|
|
"reward_std": 0.2592615157365799,
|
|
"rewards/reward_fn/mean": -0.2625,
|
|
"rewards/reward_fn/std": 0.2592615157365799,
|
|
"sampling/importance_sampling_ratio/max": 1.3269363164901733,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994530379772186,
|
|
"sampling/importance_sampling_ratio/min": 0.7182049930095673,
|
|
"sampling/sampling_logp_difference/max": 0.36123629212379454,
|
|
"sampling/sampling_logp_difference/mean": 0.021487678214907647,
|
|
"step": 9560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 273.8,
|
|
"completions/max_terminated_length": 273.8,
|
|
"completions/mean_length": 198.65,
|
|
"completions/mean_terminated_length": 198.65,
|
|
"completions/min_length": 138.6,
|
|
"completions/min_terminated_length": 138.6,
|
|
"entropy": 1.0299797058105469,
|
|
"epoch": 0.8798381906775765,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.6058125495910645,
|
|
"kl": 0.026624112448189408,
|
|
"learning_rate": 4.2339040314420785e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 41629890.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.26211473941802976,
|
|
"rewards/reward_fn/mean": 0.3125,
|
|
"rewards/reward_fn/std": 0.2621147453784943,
|
|
"sampling/importance_sampling_ratio/max": 1.2468629002571106,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998441338539124,
|
|
"sampling/importance_sampling_ratio/min": 0.7574172198772431,
|
|
"sampling/sampling_logp_difference/max": 0.29169942140579225,
|
|
"sampling/sampling_logp_difference/mean": 0.021236160211265086,
|
|
"step": 9570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.6,
|
|
"completions/max_terminated_length": 312.6,
|
|
"completions/mean_length": 222.075,
|
|
"completions/mean_terminated_length": 222.075,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 1.1217486038804054,
|
|
"epoch": 0.8807575618277098,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.760065078735352,
|
|
"kl": 0.04545778785832226,
|
|
"learning_rate": 4.232077925263231e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 41674960.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.19864802658557892,
|
|
"sampling/importance_sampling_ratio/max": 1.2878081321716308,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000447690486909,
|
|
"sampling/importance_sampling_ratio/min": 0.7819167137145996,
|
|
"sampling/sampling_logp_difference/max": 0.27499172687530515,
|
|
"sampling/sampling_logp_difference/mean": 0.021269455552101135,
|
|
"step": 9580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.1,
|
|
"completions/max_terminated_length": 315.1,
|
|
"completions/mean_length": 223.35,
|
|
"completions/mean_terminated_length": 223.35,
|
|
"completions/min_length": 139.1,
|
|
"completions/min_terminated_length": 139.1,
|
|
"entropy": 1.139740564674139,
|
|
"epoch": 0.8816769329778431,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1076013520359993,
|
|
"kl": 0.041789746819995345,
|
|
"learning_rate": 4.230250040101511e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 41720276.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.29083392918109896,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.29083394110202787,
|
|
"sampling/importance_sampling_ratio/max": 1.2562564134597778,
|
|
"sampling/importance_sampling_ratio/mean": 0.999881899356842,
|
|
"sampling/importance_sampling_ratio/min": 0.7805412769317627,
|
|
"sampling/sampling_logp_difference/max": 0.2723170816898346,
|
|
"sampling/sampling_logp_difference/mean": 0.021944673918187618,
|
|
"step": 9590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.8,
|
|
"completions/max_terminated_length": 301.8,
|
|
"completions/mean_length": 218.6,
|
|
"completions/mean_terminated_length": 218.6,
|
|
"completions/min_length": 137.3,
|
|
"completions/min_terminated_length": 137.3,
|
|
"entropy": 1.1017316907644272,
|
|
"epoch": 0.8825963041279765,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1838923692703247,
|
|
"kl": 0.049498744611628356,
|
|
"learning_rate": 4.2284203778343035e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 41765396.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.2857582807540894,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.28575828671455383,
|
|
"sampling/importance_sampling_ratio/max": 1.2418395280838013,
|
|
"sampling/importance_sampling_ratio/mean": 1.000712263584137,
|
|
"sampling/importance_sampling_ratio/min": 0.7931682825088501,
|
|
"sampling/sampling_logp_difference/max": 0.2401123285293579,
|
|
"sampling/sampling_logp_difference/mean": 0.021757611073553562,
|
|
"step": 9600
|
|
},
|
|
{
|
|
"epoch": 0.8825963041279765,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0015270506108202443,
|
|
"eval_completions/max_length": 291.6125654450262,
|
|
"eval_completions/max_terminated_length": 290.26003490401393,
|
|
"eval_completions/mean_length": 207.48058464223385,
|
|
"eval_completions/mean_terminated_length": 207.09736782426816,
|
|
"eval_completions/min_length": 136.27225130890054,
|
|
"eval_completions/min_terminated_length": 136.27225130890054,
|
|
"eval_entropy": 1.1127032290162424,
|
|
"eval_frac_reward_zero_std": 0.3699825479930192,
|
|
"eval_kl": 0.04880704622988769,
|
|
"eval_loss": 0.0025783744640648365,
|
|
"eval_num_tokens": 41765396.0,
|
|
"eval_reward": 0.07242582897033159,
|
|
"eval_reward_std": 0.30528518042639285,
|
|
"eval_rewards/reward_fn/mean": 0.07242582897033159,
|
|
"eval_rewards/reward_fn/std": 0.30528518791598175,
|
|
"eval_runtime": 1705.7337,
|
|
"eval_samples_per_second": 0.336,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2728185106530447,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.000017749805517,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7790276485691936,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2783717569256328,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02141451346728606,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 9600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.5,
|
|
"completions/max_terminated_length": 269.5,
|
|
"completions/mean_length": 197.9625,
|
|
"completions/mean_terminated_length": 197.9625,
|
|
"completions/min_length": 140.8,
|
|
"completions/min_terminated_length": 140.8,
|
|
"entropy": 1.0829307191073894,
|
|
"epoch": 0.8835156752781098,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.826517105102539,
|
|
"kl": 0.056431339180562645,
|
|
"learning_rate": 4.226588940340818e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 41806569.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.3093190997838974,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.30931911170482634,
|
|
"sampling/importance_sampling_ratio/max": 1.2439457416534423,
|
|
"sampling/importance_sampling_ratio/mean": 0.9995703756809234,
|
|
"sampling/importance_sampling_ratio/min": 0.7803526103496552,
|
|
"sampling/sampling_logp_difference/max": 0.25469855070114134,
|
|
"sampling/sampling_logp_difference/mean": 0.021404120698571207,
|
|
"step": 9610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.1,
|
|
"completions/max_terminated_length": 269.1,
|
|
"completions/mean_length": 185.1375,
|
|
"completions/mean_terminated_length": 185.1375,
|
|
"completions/min_length": 125.6,
|
|
"completions/min_terminated_length": 125.6,
|
|
"entropy": 1.0163953557610512,
|
|
"epoch": 0.8844350464282431,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.391931056976318,
|
|
"kl": 0.03501465924782678,
|
|
"learning_rate": 4.224755729502088e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 41849476.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.3021578580141068,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.3021578699350357,
|
|
"sampling/importance_sampling_ratio/max": 1.3150605797767638,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004523694515228,
|
|
"sampling/importance_sampling_ratio/min": 0.7710909605026245,
|
|
"sampling/sampling_logp_difference/max": 0.3159607410430908,
|
|
"sampling/sampling_logp_difference/mean": 0.020232264138758182,
|
|
"step": 9620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.5,
|
|
"completions/max_terminated_length": 295.5,
|
|
"completions/mean_length": 207.325,
|
|
"completions/mean_terminated_length": 207.325,
|
|
"completions/min_length": 119.2,
|
|
"completions/min_terminated_length": 119.2,
|
|
"entropy": 1.0288881592452526,
|
|
"epoch": 0.8853544175783764,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.479791641235352,
|
|
"kl": 0.046140060760080814,
|
|
"learning_rate": 4.222920747200967e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 41894230.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.3627713471651077,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.3627713590860367,
|
|
"sampling/importance_sampling_ratio/max": 1.2772618174552917,
|
|
"sampling/importance_sampling_ratio/mean": 0.999645447731018,
|
|
"sampling/importance_sampling_ratio/min": 0.7572857320308686,
|
|
"sampling/sampling_logp_difference/max": 0.29203481674194337,
|
|
"sampling/sampling_logp_difference/mean": 0.02117515280842781,
|
|
"step": 9630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.8,
|
|
"completions/max_terminated_length": 309.8,
|
|
"completions/mean_length": 208.975,
|
|
"completions/mean_terminated_length": 208.975,
|
|
"completions/min_length": 127.4,
|
|
"completions/min_terminated_length": 127.4,
|
|
"entropy": 1.1508677646517753,
|
|
"epoch": 0.8862737887285097,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.340304851531982,
|
|
"kl": 0.06683948591817171,
|
|
"learning_rate": 4.221083995322131e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 41935668.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.321952098608017,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.32195210456848145,
|
|
"sampling/importance_sampling_ratio/max": 1.2640284299850464,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002631366252899,
|
|
"sampling/importance_sampling_ratio/min": 0.7422779858112335,
|
|
"sampling/sampling_logp_difference/max": 0.3229871392250061,
|
|
"sampling/sampling_logp_difference/mean": 0.022663329914212226,
|
|
"step": 9640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 294.0,
|
|
"completions/max_terminated_length": 294.0,
|
|
"completions/mean_length": 222.2125,
|
|
"completions/mean_terminated_length": 222.2125,
|
|
"completions/min_length": 147.3,
|
|
"completions/min_terminated_length": 147.3,
|
|
"entropy": 1.1123330168426038,
|
|
"epoch": 0.887193159878643,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.3753557205200195,
|
|
"kl": 0.04909877778845839,
|
|
"learning_rate": 4.219245475752069e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 41981845.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.18685350716114044,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.18685351312160492,
|
|
"sampling/importance_sampling_ratio/max": 1.2773982405662536,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004447519779205,
|
|
"sampling/importance_sampling_ratio/min": 0.7566041707992553,
|
|
"sampling/sampling_logp_difference/max": 0.29627662897109985,
|
|
"sampling/sampling_logp_difference/mean": 0.021852345764636995,
|
|
"step": 9650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.6,
|
|
"completions/max_terminated_length": 289.6,
|
|
"completions/mean_length": 217.7125,
|
|
"completions/mean_terminated_length": 217.7125,
|
|
"completions/min_length": 136.4,
|
|
"completions/min_terminated_length": 136.4,
|
|
"entropy": 1.0822629570960998,
|
|
"epoch": 0.8881125310287763,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.162141799926758,
|
|
"kl": 0.05092568336986005,
|
|
"learning_rate": 4.217405190379089e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 42025438.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.4968440443277359,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.4968440443277359,
|
|
"sampling/importance_sampling_ratio/max": 1.2648176193237304,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001706898212432,
|
|
"sampling/importance_sampling_ratio/min": 0.7700137555599212,
|
|
"sampling/sampling_logp_difference/max": 0.2854828894138336,
|
|
"sampling/sampling_logp_difference/mean": 0.02105266060680151,
|
|
"step": 9660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 346.8,
|
|
"completions/max_terminated_length": 346.8,
|
|
"completions/mean_length": 233.0125,
|
|
"completions/mean_terminated_length": 233.0125,
|
|
"completions/min_length": 146.3,
|
|
"completions/min_terminated_length": 146.3,
|
|
"entropy": 1.152537815272808,
|
|
"epoch": 0.8890319021789096,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 8.657646179199219,
|
|
"kl": 0.05522806823719293,
|
|
"learning_rate": 4.215563141093312e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 42073471.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.3093190997838974,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.30931911170482634,
|
|
"sampling/importance_sampling_ratio/max": 1.2799705028533936,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006804287433624,
|
|
"sampling/importance_sampling_ratio/min": 0.7892005026340485,
|
|
"sampling/sampling_logp_difference/max": 0.2680843472480774,
|
|
"sampling/sampling_logp_difference/mean": 0.0221128661185503,
|
|
"step": 9670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 254.8,
|
|
"completions/max_terminated_length": 254.8,
|
|
"completions/mean_length": 186.4375,
|
|
"completions/mean_terminated_length": 186.4375,
|
|
"completions/min_length": 115.5,
|
|
"completions/min_terminated_length": 115.5,
|
|
"entropy": 0.9961151644587517,
|
|
"epoch": 0.8899512733290429,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.703009128570557,
|
|
"kl": 0.05263791738543659,
|
|
"learning_rate": 4.213719329786669e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 42113378.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.2974700778722763,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.29747008383274076,
|
|
"sampling/importance_sampling_ratio/max": 1.250830590724945,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997282445430755,
|
|
"sampling/importance_sampling_ratio/min": 0.7526599764823914,
|
|
"sampling/sampling_logp_difference/max": 0.30122084617614747,
|
|
"sampling/sampling_logp_difference/mean": 0.02062029894441366,
|
|
"step": 9680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 307.6,
|
|
"completions/max_terminated_length": 307.6,
|
|
"completions/mean_length": 233.775,
|
|
"completions/mean_terminated_length": 233.775,
|
|
"completions/min_length": 163.5,
|
|
"completions/min_terminated_length": 163.5,
|
|
"entropy": 1.1191049337387085,
|
|
"epoch": 0.8908706444791762,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.562215805053711,
|
|
"kl": 0.04675295646302402,
|
|
"learning_rate": 4.211873758352905e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 42161408.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.3878450483083725,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.387845054268837,
|
|
"sampling/importance_sampling_ratio/max": 1.2485342264175414,
|
|
"sampling/importance_sampling_ratio/mean": 1.000311905145645,
|
|
"sampling/importance_sampling_ratio/min": 0.7557773053646087,
|
|
"sampling/sampling_logp_difference/max": 0.2908876299858093,
|
|
"sampling/sampling_logp_difference/mean": 0.02112082876265049,
|
|
"step": 9690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.1,
|
|
"completions/max_terminated_length": 297.1,
|
|
"completions/mean_length": 230.5125,
|
|
"completions/mean_terminated_length": 230.5125,
|
|
"completions/min_length": 168.3,
|
|
"completions/min_terminated_length": 168.3,
|
|
"entropy": 1.1685070395469666,
|
|
"epoch": 0.8917900156293096,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.073295593261719,
|
|
"kl": 0.060053869453258815,
|
|
"learning_rate": 4.2100264286875655e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 42208641.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.4228512436151505,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.4228512614965439,
|
|
"sampling/importance_sampling_ratio/max": 1.2690693855285644,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993915319442749,
|
|
"sampling/importance_sampling_ratio/min": 0.7842618942260742,
|
|
"sampling/sampling_logp_difference/max": 0.2743223309516907,
|
|
"sampling/sampling_logp_difference/mean": 0.022326854430139063,
|
|
"step": 9700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.5,
|
|
"completions/max_terminated_length": 296.5,
|
|
"completions/mean_length": 224.625,
|
|
"completions/mean_terminated_length": 224.625,
|
|
"completions/min_length": 168.8,
|
|
"completions/min_terminated_length": 168.8,
|
|
"entropy": 1.0453823491930962,
|
|
"epoch": 0.8927093867794429,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.0743865966796875,
|
|
"kl": 0.041981905058491975,
|
|
"learning_rate": 4.208177342688009e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 42255699.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.3716338902711868,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.3716339021921158,
|
|
"sampling/importance_sampling_ratio/max": 1.2806673288345336,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004289627075196,
|
|
"sampling/importance_sampling_ratio/min": 0.7232143551111221,
|
|
"sampling/sampling_logp_difference/max": 0.3720430850982666,
|
|
"sampling/sampling_logp_difference/mean": 0.020231438241899012,
|
|
"step": 9710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.1,
|
|
"completions/max_terminated_length": 269.1,
|
|
"completions/mean_length": 192.2625,
|
|
"completions/mean_terminated_length": 192.2625,
|
|
"completions/min_length": 133.7,
|
|
"completions/min_terminated_length": 133.7,
|
|
"entropy": 1.163520159572363,
|
|
"epoch": 0.8936287579295762,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 8.238802909851074,
|
|
"kl": 0.050715148559538646,
|
|
"learning_rate": 4.206326502253395e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 42299448.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.1977891743183136,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.19778918027877807,
|
|
"sampling/importance_sampling_ratio/max": 1.2369333624839782,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993122518062592,
|
|
"sampling/importance_sampling_ratio/min": 0.800760930776596,
|
|
"sampling/sampling_logp_difference/max": 0.23701193332672119,
|
|
"sampling/sampling_logp_difference/mean": 0.022435952723026276,
|
|
"step": 9720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.5,
|
|
"completions/max_terminated_length": 276.5,
|
|
"completions/mean_length": 192.0125,
|
|
"completions/mean_terminated_length": 192.0125,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 1.1158318415284156,
|
|
"epoch": 0.8945481290797095,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.269508361816406,
|
|
"kl": 0.048604181536938997,
|
|
"learning_rate": 4.204473909284683e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 42339105.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.41999801993370056,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.4199980318546295,
|
|
"sampling/importance_sampling_ratio/max": 1.2698224186897278,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997045993804932,
|
|
"sampling/importance_sampling_ratio/min": 0.7689110994338989,
|
|
"sampling/sampling_logp_difference/max": 0.2866933584213257,
|
|
"sampling/sampling_logp_difference/mean": 0.02233223579823971,
|
|
"step": 9730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.8,
|
|
"completions/max_terminated_length": 286.8,
|
|
"completions/mean_length": 208.2875,
|
|
"completions/mean_terminated_length": 208.2875,
|
|
"completions/min_length": 138.5,
|
|
"completions/min_terminated_length": 138.5,
|
|
"entropy": 1.1071766138076782,
|
|
"epoch": 0.8954675002298428,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12204127013683319,
|
|
"kl": 0.05414634409826249,
|
|
"learning_rate": 4.202619565684637e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 42384864.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.32480532228946685,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.3248053342103958,
|
|
"sampling/importance_sampling_ratio/max": 1.278862726688385,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000852704048158,
|
|
"sampling/importance_sampling_ratio/min": 0.7806693851947785,
|
|
"sampling/sampling_logp_difference/max": 0.2705940842628479,
|
|
"sampling/sampling_logp_difference/mean": 0.021467828005552293,
|
|
"step": 9740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 243.7,
|
|
"completions/max_terminated_length": 243.7,
|
|
"completions/mean_length": 177.5375,
|
|
"completions/mean_terminated_length": 177.5375,
|
|
"completions/min_length": 107.5,
|
|
"completions/min_terminated_length": 107.5,
|
|
"entropy": 1.036444703489542,
|
|
"epoch": 0.8963868713799761,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.280074805021286,
|
|
"kl": 0.0429881299380213,
|
|
"learning_rate": 4.200763473357815e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 42425307.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.29291952252388,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.2929195284843445,
|
|
"sampling/importance_sampling_ratio/max": 1.2604297876358033,
|
|
"sampling/importance_sampling_ratio/mean": 0.99969522356987,
|
|
"sampling/importance_sampling_ratio/min": 0.7613808035850524,
|
|
"sampling/sampling_logp_difference/max": 0.2779817461967468,
|
|
"sampling/sampling_logp_difference/mean": 0.021500294841825963,
|
|
"step": 9750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 278.9,
|
|
"completions/max_terminated_length": 269.6,
|
|
"completions/mean_length": 197.5875,
|
|
"completions/mean_terminated_length": 194.57857360839844,
|
|
"completions/min_length": 119.8,
|
|
"completions/min_terminated_length": 119.8,
|
|
"entropy": 1.100955756008625,
|
|
"epoch": 0.8973062425301094,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.21760457754135132,
|
|
"kl": 0.051878090959507975,
|
|
"learning_rate": 4.198905634210574e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 42468834.0,
|
|
"reward": -0.11875,
|
|
"reward_std": 0.32717646956443786,
|
|
"rewards/reward_fn/mean": -0.11875,
|
|
"rewards/reward_fn/std": 0.3271764814853668,
|
|
"sampling/importance_sampling_ratio/max": 1.2780219674110413,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996322333812714,
|
|
"sampling/importance_sampling_ratio/min": 0.7522034108638763,
|
|
"sampling/sampling_logp_difference/max": 0.3005282640457153,
|
|
"sampling/sampling_logp_difference/mean": 0.021998685225844382,
|
|
"step": 9760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 249.5,
|
|
"completions/max_terminated_length": 249.5,
|
|
"completions/mean_length": 185.1375,
|
|
"completions/mean_terminated_length": 185.1375,
|
|
"completions/min_length": 125.0,
|
|
"completions/min_terminated_length": 125.0,
|
|
"entropy": 1.058626901358366,
|
|
"epoch": 0.8982256136802427,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.14348600804805756,
|
|
"kl": 0.04166007454041391,
|
|
"learning_rate": 4.197046050151061e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 42507109.0,
|
|
"reward": -0.0625,
|
|
"reward_std": 0.3475900113582611,
|
|
"rewards/reward_fn/mean": -0.0625,
|
|
"rewards/reward_fn/std": 0.34759002923965454,
|
|
"sampling/importance_sampling_ratio/max": 1.2821545600891113,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005722522735596,
|
|
"sampling/importance_sampling_ratio/min": 0.7877614319324493,
|
|
"sampling/sampling_logp_difference/max": 0.2655537486076355,
|
|
"sampling/sampling_logp_difference/mean": 0.021121001057326793,
|
|
"step": 9770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.8,
|
|
"completions/max_terminated_length": 280.8,
|
|
"completions/mean_length": 196.075,
|
|
"completions/mean_terminated_length": 196.075,
|
|
"completions/min_length": 126.5,
|
|
"completions/min_terminated_length": 126.5,
|
|
"entropy": 1.1112434238195419,
|
|
"epoch": 0.899144984830376,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.007650375366211,
|
|
"kl": 0.042061880818801,
|
|
"learning_rate": 4.19518472308922e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 42547011.0,
|
|
"reward": 0.4875,
|
|
"reward_std": 0.2095758467912674,
|
|
"rewards/reward_fn/mean": 0.4875,
|
|
"rewards/reward_fn/std": 0.20957585871219636,
|
|
"sampling/importance_sampling_ratio/max": 1.2591874837875365,
|
|
"sampling/importance_sampling_ratio/mean": 0.999758917093277,
|
|
"sampling/importance_sampling_ratio/min": 0.7797321140766144,
|
|
"sampling/sampling_logp_difference/max": 0.27152804136276243,
|
|
"sampling/sampling_logp_difference/mean": 0.02207108736038208,
|
|
"step": 9780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.0,
|
|
"completions/max_terminated_length": 280.0,
|
|
"completions/mean_length": 203.2125,
|
|
"completions/mean_terminated_length": 203.2125,
|
|
"completions/min_length": 124.9,
|
|
"completions/min_terminated_length": 124.9,
|
|
"entropy": 1.1558429151773453,
|
|
"epoch": 0.9000643559805094,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 9.507164001464844,
|
|
"kl": 0.05067404091823846,
|
|
"learning_rate": 4.193321654936784e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 42586884.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.20295216739177704,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.20295217931270598,
|
|
"sampling/importance_sampling_ratio/max": 1.2489335656166076,
|
|
"sampling/importance_sampling_ratio/mean": 1.000390750169754,
|
|
"sampling/importance_sampling_ratio/min": 0.7842198371887207,
|
|
"sampling/sampling_logp_difference/max": 0.2511386752128601,
|
|
"sampling/sampling_logp_difference/mean": 0.022299526445567607,
|
|
"step": 9790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 284.0,
|
|
"completions/max_terminated_length": 284.0,
|
|
"completions/mean_length": 201.85,
|
|
"completions/mean_terminated_length": 201.85,
|
|
"completions/min_length": 131.3,
|
|
"completions/min_terminated_length": 131.3,
|
|
"entropy": 1.1762912645936012,
|
|
"epoch": 0.9009837271306427,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12105719745159149,
|
|
"kl": 0.05483786647673696,
|
|
"learning_rate": 4.191456847607271e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 42628088.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.26681036651134493,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.26681036651134493,
|
|
"sampling/importance_sampling_ratio/max": 1.2537169218063355,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003124713897704,
|
|
"sampling/importance_sampling_ratio/min": 0.7906040370464325,
|
|
"sampling/sampling_logp_difference/max": 0.25084388256073,
|
|
"sampling/sampling_logp_difference/mean": 0.022775582410395144,
|
|
"step": 9800
|
|
},
|
|
{
|
|
"epoch": 0.9009837271306427,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0008726003490401396,
|
|
"eval_completions/max_length": 280.74869109947645,
|
|
"eval_completions/max_terminated_length": 279.5602094240838,
|
|
"eval_completions/mean_length": 197.31239092495636,
|
|
"eval_completions/mean_terminated_length": 197.07725634083906,
|
|
"eval_completions/min_length": 127.43979057591623,
|
|
"eval_completions/min_terminated_length": 127.43979057591623,
|
|
"eval_entropy": 1.1062782172237182,
|
|
"eval_frac_reward_zero_std": 0.34904013961605584,
|
|
"eval_kl": 0.047489816098625126,
|
|
"eval_loss": 0.0024971631355583668,
|
|
"eval_num_tokens": 42628088.0,
|
|
"eval_reward": 0.07569808027923211,
|
|
"eval_reward_std": 0.31951899615881957,
|
|
"eval_rewards/reward_fn/mean": 0.07569808027923211,
|
|
"eval_rewards/reward_fn/std": 0.31951900489667323,
|
|
"eval_runtime": 1652.3955,
|
|
"eval_samples_per_second": 0.347,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2657346534063263,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999833720724828,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7812690228795922,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2738112529534944,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02136060033143087,
|
|
"eval_steps_per_second": 0.044,
|
|
"step": 9800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.2,
|
|
"completions/max_terminated_length": 279.2,
|
|
"completions/mean_length": 207.3625,
|
|
"completions/mean_terminated_length": 207.3625,
|
|
"completions/min_length": 148.7,
|
|
"completions/min_terminated_length": 148.7,
|
|
"entropy": 1.0698274560272694,
|
|
"epoch": 0.901903098280776,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.936789512634277,
|
|
"kl": 0.058363816619385035,
|
|
"learning_rate": 4.189590303015989e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 42673981.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.4608172684907913,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.4608172863721848,
|
|
"sampling/importance_sampling_ratio/max": 1.3146578550338746,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000967919826507,
|
|
"sampling/importance_sampling_ratio/min": 0.7536275386810303,
|
|
"sampling/sampling_logp_difference/max": 0.3268724262714386,
|
|
"sampling/sampling_logp_difference/mean": 0.02157493242993951,
|
|
"step": 9810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.0,
|
|
"completions/max_terminated_length": 293.0,
|
|
"completions/mean_length": 196.1625,
|
|
"completions/mean_terminated_length": 196.1625,
|
|
"completions/min_length": 131.4,
|
|
"completions/min_terminated_length": 131.4,
|
|
"entropy": 0.9949974793940782,
|
|
"epoch": 0.9028224694309093,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.13990524411201477,
|
|
"kl": 0.0469113961677067,
|
|
"learning_rate": 4.187722023080028e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 42714114.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.2112731784582138,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.2112731844186783,
|
|
"sampling/importance_sampling_ratio/max": 1.3319814682006836,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006243526935577,
|
|
"sampling/importance_sampling_ratio/min": 0.7772722601890564,
|
|
"sampling/sampling_logp_difference/max": 0.3133630394935608,
|
|
"sampling/sampling_logp_difference/mean": 0.020639171823859215,
|
|
"step": 9820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.7,
|
|
"completions/max_terminated_length": 295.7,
|
|
"completions/mean_length": 212.175,
|
|
"completions/mean_terminated_length": 212.175,
|
|
"completions/min_length": 143.3,
|
|
"completions/min_terminated_length": 143.3,
|
|
"entropy": 1.155884674936533,
|
|
"epoch": 0.9037418405810426,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.2534589767456055,
|
|
"kl": 0.07070289278635755,
|
|
"learning_rate": 4.1858520097182635e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 42758664.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.29291952252388,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.2929195284843445,
|
|
"sampling/importance_sampling_ratio/max": 1.2690081477165223,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004143595695496,
|
|
"sampling/importance_sampling_ratio/min": 0.7489427924156189,
|
|
"sampling/sampling_logp_difference/max": 0.3149022817611694,
|
|
"sampling/sampling_logp_difference/mean": 0.02249901872128248,
|
|
"step": 9830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 266.4,
|
|
"completions/max_terminated_length": 266.4,
|
|
"completions/mean_length": 187.575,
|
|
"completions/mean_terminated_length": 187.575,
|
|
"completions/min_length": 115.6,
|
|
"completions/min_terminated_length": 115.6,
|
|
"entropy": 1.0035645239055158,
|
|
"epoch": 0.9046612117311759,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 7.56233549118042,
|
|
"kl": 0.05291398569825105,
|
|
"learning_rate": 4.183980264851345e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 42800046.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.2276727557182312,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.22767276763916017,
|
|
"sampling/importance_sampling_ratio/max": 1.2687315464019775,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998767256736756,
|
|
"sampling/importance_sampling_ratio/min": 0.7907434582710267,
|
|
"sampling/sampling_logp_difference/max": 0.26653594970703126,
|
|
"sampling/sampling_logp_difference/mean": 0.020193965174257755,
|
|
"step": 9840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.0,
|
|
"completions/max_terminated_length": 282.0,
|
|
"completions/mean_length": 205.5125,
|
|
"completions/mean_terminated_length": 205.5125,
|
|
"completions/min_length": 146.5,
|
|
"completions/min_terminated_length": 146.5,
|
|
"entropy": 1.1803817585110665,
|
|
"epoch": 0.9055805828813092,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.895437240600586,
|
|
"kl": 0.06135212822118774,
|
|
"learning_rate": 4.182106790401708e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 42844935.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.2808319330215454,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.2808319330215454,
|
|
"sampling/importance_sampling_ratio/max": 1.2608348369598388,
|
|
"sampling/importance_sampling_ratio/mean": 1.000287228822708,
|
|
"sampling/importance_sampling_ratio/min": 0.8008495986461639,
|
|
"sampling/sampling_logp_difference/max": 0.2604467451572418,
|
|
"sampling/sampling_logp_difference/mean": 0.022341980040073393,
|
|
"step": 9850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 249.0,
|
|
"completions/max_terminated_length": 249.0,
|
|
"completions/mean_length": 176.9125,
|
|
"completions/mean_terminated_length": 176.9125,
|
|
"completions/min_length": 113.1,
|
|
"completions/min_terminated_length": 113.1,
|
|
"entropy": 1.1014343447983266,
|
|
"epoch": 0.9064999540314425,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.12081068754196167,
|
|
"kl": 0.06852991638006642,
|
|
"learning_rate": 4.1802315882935605e-06,
|
|
"loss": 0.0037,
|
|
"num_tokens": 42886400.0,
|
|
"reward": -0.1125,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/reward_fn/mean": -0.1125,
|
|
"rewards/reward_fn/std": 0.19864802658557892,
|
|
"sampling/importance_sampling_ratio/max": 1.275366449356079,
|
|
"sampling/importance_sampling_ratio/mean": 0.9991794168949127,
|
|
"sampling/importance_sampling_ratio/min": 0.7355645060539245,
|
|
"sampling/sampling_logp_difference/max": 0.37657909393310546,
|
|
"sampling/sampling_logp_difference/mean": 0.021964114159345627,
|
|
"step": 9860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.1,
|
|
"completions/max_terminated_length": 304.1,
|
|
"completions/mean_length": 215.0375,
|
|
"completions/mean_terminated_length": 215.0375,
|
|
"completions/min_length": 142.3,
|
|
"completions/min_terminated_length": 142.3,
|
|
"entropy": 1.0922230079770088,
|
|
"epoch": 0.9074193251815758,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.570449352264404,
|
|
"kl": 0.05736344971228391,
|
|
"learning_rate": 4.178354660452885e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 42934603.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.22220885157585143,
|
|
"sampling/importance_sampling_ratio/max": 1.2846588969230652,
|
|
"sampling/importance_sampling_ratio/mean": 1.00008225440979,
|
|
"sampling/importance_sampling_ratio/min": 0.7467437446117401,
|
|
"sampling/sampling_logp_difference/max": 0.2975830793380737,
|
|
"sampling/sampling_logp_difference/mean": 0.021936732903122903,
|
|
"step": 9870
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 266.0,
|
|
"completions/max_terminated_length": 266.0,
|
|
"completions/mean_length": 196.95,
|
|
"completions/mean_terminated_length": 196.95,
|
|
"completions/min_length": 130.1,
|
|
"completions/min_terminated_length": 130.1,
|
|
"entropy": 1.071858535706997,
|
|
"epoch": 0.9083386963317092,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.13019002974033356,
|
|
"kl": 0.057886114652501416,
|
|
"learning_rate": 4.176476008807435e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 42975351.0,
|
|
"reward": 0.13125,
|
|
"reward_std": 0.40080119073390963,
|
|
"rewards/reward_fn/mean": 0.13125,
|
|
"rewards/reward_fn/std": 0.40080119073390963,
|
|
"sampling/importance_sampling_ratio/max": 1.2981379866600036,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002736091613769,
|
|
"sampling/importance_sampling_ratio/min": 0.7947067618370056,
|
|
"sampling/sampling_logp_difference/max": 0.2724072337150574,
|
|
"sampling/sampling_logp_difference/mean": 0.021525019966065883,
|
|
"step": 9880
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.0,
|
|
"completions/max_terminated_length": 283.0,
|
|
"completions/mean_length": 203.8625,
|
|
"completions/mean_terminated_length": 203.8625,
|
|
"completions/min_length": 132.2,
|
|
"completions/min_terminated_length": 132.2,
|
|
"entropy": 1.1793571673333645,
|
|
"epoch": 0.9092580674818425,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.12472919374704361,
|
|
"kl": 0.053053760924376546,
|
|
"learning_rate": 4.174595635286739e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 43021732.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.20958369374275207,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.20958369374275207,
|
|
"sampling/importance_sampling_ratio/max": 1.2728296995162964,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999460637569427,
|
|
"sampling/importance_sampling_ratio/min": 0.7885064840316772,
|
|
"sampling/sampling_logp_difference/max": 0.2676731765270233,
|
|
"sampling/sampling_logp_difference/mean": 0.022822683677077293,
|
|
"step": 9890
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.0,
|
|
"completions/max_terminated_length": 283.0,
|
|
"completions/mean_length": 207.425,
|
|
"completions/mean_terminated_length": 207.425,
|
|
"completions/min_length": 137.2,
|
|
"completions/min_terminated_length": 137.2,
|
|
"entropy": 1.11990627348423,
|
|
"epoch": 0.9101774386319758,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.911685943603516,
|
|
"kl": 0.06903767628828064,
|
|
"learning_rate": 4.17271354182209e-06,
|
|
"loss": 0.0039,
|
|
"num_tokens": 43066350.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.28659676015377045,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.2865967661142349,
|
|
"sampling/importance_sampling_ratio/max": 1.2657966613769531,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005126535892486,
|
|
"sampling/importance_sampling_ratio/min": 0.797953587770462,
|
|
"sampling/sampling_logp_difference/max": 0.2551021337509155,
|
|
"sampling/sampling_logp_difference/mean": 0.021488823182880878,
|
|
"step": 9900
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.6,
|
|
"completions/max_terminated_length": 255.6,
|
|
"completions/mean_length": 188.6625,
|
|
"completions/mean_terminated_length": 188.6625,
|
|
"completions/min_length": 130.6,
|
|
"completions/min_terminated_length": 130.6,
|
|
"entropy": 1.0584544934332372,
|
|
"epoch": 0.911096809782109,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.512333869934082,
|
|
"kl": 0.06185520152794197,
|
|
"learning_rate": 4.170829730346549e-06,
|
|
"loss": 0.0033,
|
|
"num_tokens": 43105163.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.2394672751426697,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.23946728110313414,
|
|
"sampling/importance_sampling_ratio/max": 1.2746208310127258,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999439895153046,
|
|
"sampling/importance_sampling_ratio/min": 0.769288969039917,
|
|
"sampling/sampling_logp_difference/max": 0.290193772315979,
|
|
"sampling/sampling_logp_difference/mean": 0.021467641554772855,
|
|
"step": 9910
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.7,
|
|
"completions/max_terminated_length": 270.7,
|
|
"completions/mean_length": 187.6375,
|
|
"completions/mean_terminated_length": 187.6375,
|
|
"completions/min_length": 126.7,
|
|
"completions/min_terminated_length": 126.7,
|
|
"entropy": 1.0328537292778492,
|
|
"epoch": 0.9120161809322423,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.11428701877593994,
|
|
"kl": 0.05851017193053849,
|
|
"learning_rate": 4.168944202794942e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 43146038.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/reward_fn/mean": 0.4125,
|
|
"rewards/reward_fn/std": 0.22220885157585143,
|
|
"sampling/importance_sampling_ratio/max": 1.2857640504837036,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997570037841796,
|
|
"sampling/importance_sampling_ratio/min": 0.770370090007782,
|
|
"sampling/sampling_logp_difference/max": 0.28543384075164796,
|
|
"sampling/sampling_logp_difference/mean": 0.0211546678096056,
|
|
"step": 9920
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 270.3,
|
|
"completions/max_terminated_length": 270.3,
|
|
"completions/mean_length": 184.525,
|
|
"completions/mean_terminated_length": 184.525,
|
|
"completions/min_length": 121.0,
|
|
"completions/min_terminated_length": 121.0,
|
|
"entropy": 1.0869535781443118,
|
|
"epoch": 0.9129355520823756,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.079781532287598,
|
|
"kl": 0.06194143147440627,
|
|
"learning_rate": 4.167056961103857e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 43187976.0,
|
|
"reward": -0.15,
|
|
"reward_std": 0.41369563043117524,
|
|
"rewards/reward_fn/mean": -0.15,
|
|
"rewards/reward_fn/std": 0.4136956363916397,
|
|
"sampling/importance_sampling_ratio/max": 1.2453197360038757,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996575653553009,
|
|
"sampling/importance_sampling_ratio/min": 0.7913166224956513,
|
|
"sampling/sampling_logp_difference/max": 0.24218196868896485,
|
|
"sampling/sampling_logp_difference/mean": 0.02168115172535181,
|
|
"step": 9930
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.0,
|
|
"completions/max_terminated_length": 274.0,
|
|
"completions/mean_length": 192.6375,
|
|
"completions/mean_terminated_length": 192.6375,
|
|
"completions/min_length": 120.1,
|
|
"completions/min_terminated_length": 120.1,
|
|
"entropy": 1.0639159835875034,
|
|
"epoch": 0.9138549232325089,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12257706373929977,
|
|
"kl": 0.05870253099128604,
|
|
"learning_rate": 4.165168007211641e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 43229947.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.27851431667804716,
|
|
"rewards/reward_fn/mean": 0.3375,
|
|
"rewards/reward_fn/std": 0.2785143285989761,
|
|
"sampling/importance_sampling_ratio/max": 1.2802238941192627,
|
|
"sampling/importance_sampling_ratio/mean": 1.000627338886261,
|
|
"sampling/importance_sampling_ratio/min": 0.8046356737613678,
|
|
"sampling/sampling_logp_difference/max": 0.2600499868392944,
|
|
"sampling/sampling_logp_difference/mean": 0.02083715479820967,
|
|
"step": 9940
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.1,
|
|
"completions/max_terminated_length": 286.1,
|
|
"completions/mean_length": 198.7625,
|
|
"completions/mean_terminated_length": 198.7625,
|
|
"completions/min_length": 126.1,
|
|
"completions/min_terminated_length": 126.1,
|
|
"entropy": 1.078206367045641,
|
|
"epoch": 0.9147742943826422,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 6.023697853088379,
|
|
"kl": 0.041168544196989384,
|
|
"learning_rate": 4.163277343058402e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 43272808.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.18138959705829621,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.18138959705829621,
|
|
"sampling/importance_sampling_ratio/max": 1.2977841377258301,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998321771621704,
|
|
"sampling/importance_sampling_ratio/min": 0.7729258179664612,
|
|
"sampling/sampling_logp_difference/max": 0.29594180583953855,
|
|
"sampling/sampling_logp_difference/mean": 0.021799837611615656,
|
|
"step": 9950
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 339.7,
|
|
"completions/max_terminated_length": 339.7,
|
|
"completions/mean_length": 242.475,
|
|
"completions/mean_terminated_length": 242.475,
|
|
"completions/min_length": 160.3,
|
|
"completions/min_terminated_length": 160.3,
|
|
"entropy": 1.1220299161970615,
|
|
"epoch": 0.9156936655327755,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.079929828643799,
|
|
"kl": 0.04531771424226463,
|
|
"learning_rate": 4.1613849705860045e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 43317446.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.307040074467659,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.30704008638858793,
|
|
"sampling/importance_sampling_ratio/max": 1.328687596321106,
|
|
"sampling/importance_sampling_ratio/mean": 1.000167453289032,
|
|
"sampling/importance_sampling_ratio/min": 0.7499541193246841,
|
|
"sampling/sampling_logp_difference/max": 0.3055580735206604,
|
|
"sampling/sampling_logp_difference/mean": 0.0210553215816617,
|
|
"step": 9960
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.8,
|
|
"completions/max_terminated_length": 298.8,
|
|
"completions/mean_length": 207.5625,
|
|
"completions/mean_terminated_length": 207.5625,
|
|
"completions/min_length": 138.3,
|
|
"completions/min_terminated_length": 138.3,
|
|
"entropy": 1.0736882220953703,
|
|
"epoch": 0.9166130366829089,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.911686897277832,
|
|
"kl": 0.04477341107558459,
|
|
"learning_rate": 4.159490891738065e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 43359507.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.3228109508752823,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.3228109508752823,
|
|
"sampling/importance_sampling_ratio/max": 1.2542143225669862,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001445591449738,
|
|
"sampling/importance_sampling_ratio/min": 0.7846239686012269,
|
|
"sampling/sampling_logp_difference/max": 0.2642161905765533,
|
|
"sampling/sampling_logp_difference/mean": 0.02142866849899292,
|
|
"step": 9970
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.0,
|
|
"completions/max_terminated_length": 286.0,
|
|
"completions/mean_length": 203.6625,
|
|
"completions/mean_terminated_length": 203.6625,
|
|
"completions/min_length": 128.2,
|
|
"completions/min_terminated_length": 128.2,
|
|
"entropy": 1.0757963739335537,
|
|
"epoch": 0.9175324078330422,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.130559504032135,
|
|
"kl": 0.048978907556738706,
|
|
"learning_rate": 4.157595108459956e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 43402576.0,
|
|
"reward": -0.2125,
|
|
"reward_std": 0.3728685349225998,
|
|
"rewards/reward_fn/mean": -0.2125,
|
|
"rewards/reward_fn/std": 0.37286854684352877,
|
|
"sampling/importance_sampling_ratio/max": 1.3121480703353883,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003035902976989,
|
|
"sampling/importance_sampling_ratio/min": 0.8058617532253265,
|
|
"sampling/sampling_logp_difference/max": 0.2850976288318634,
|
|
"sampling/sampling_logp_difference/mean": 0.02138405852019787,
|
|
"step": 9980
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 333.7,
|
|
"completions/max_terminated_length": 333.7,
|
|
"completions/mean_length": 226.175,
|
|
"completions/mean_terminated_length": 226.175,
|
|
"completions/min_length": 149.9,
|
|
"completions/min_terminated_length": 149.9,
|
|
"entropy": 1.109637549519539,
|
|
"epoch": 0.9184517789831755,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.069444179534912,
|
|
"kl": 0.03635451651061885,
|
|
"learning_rate": 4.155697622698798e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 43446054.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.3595789670944214,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.35957897305488584,
|
|
"sampling/importance_sampling_ratio/max": 1.2771041750907899,
|
|
"sampling/importance_sampling_ratio/mean": 0.9994903087615967,
|
|
"sampling/importance_sampling_ratio/min": 0.7605353116989135,
|
|
"sampling/sampling_logp_difference/max": 0.29628159999847414,
|
|
"sampling/sampling_logp_difference/mean": 0.02149726580828428,
|
|
"step": 9990
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.3,
|
|
"completions/max_terminated_length": 269.3,
|
|
"completions/mean_length": 201.1125,
|
|
"completions/mean_terminated_length": 201.1125,
|
|
"completions/min_length": 137.3,
|
|
"completions/min_terminated_length": 137.3,
|
|
"entropy": 1.1079287990927695,
|
|
"epoch": 0.9193711501333088,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 10.081376075744629,
|
|
"kl": 0.047466059145517644,
|
|
"learning_rate": 4.153798436403461e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 43487727.0,
|
|
"reward": -0.0375,
|
|
"reward_std": 0.39962208569049834,
|
|
"rewards/reward_fn/mean": -0.0375,
|
|
"rewards/reward_fn/std": 0.3996220976114273,
|
|
"sampling/importance_sampling_ratio/max": 1.2673521995544434,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002014219760895,
|
|
"sampling/importance_sampling_ratio/min": 0.7535767048597336,
|
|
"sampling/sampling_logp_difference/max": 0.3077139317989349,
|
|
"sampling/sampling_logp_difference/mean": 0.022001378051936627,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 0.9193711501333088,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 293.19895287958116,
|
|
"eval_completions/max_terminated_length": 291.32460732984293,
|
|
"eval_completions/mean_length": 206.18848167539267,
|
|
"eval_completions/mean_terminated_length": 205.8286900012697,
|
|
"eval_completions/min_length": 132.62303664921467,
|
|
"eval_completions/min_terminated_length": 132.62303664921467,
|
|
"eval_entropy": 1.106627268056803,
|
|
"eval_frac_reward_zero_std": 0.33507853403141363,
|
|
"eval_kl": 0.04242372071292466,
|
|
"eval_loss": 0.002180905546993017,
|
|
"eval_num_tokens": 43487727.0,
|
|
"eval_reward": 0.07907940663176265,
|
|
"eval_reward_std": 0.3361895477064409,
|
|
"eval_rewards/reward_fn/mean": 0.07907940663176265,
|
|
"eval_rewards/reward_fn/std": 0.336189557068427,
|
|
"eval_runtime": 1714.9811,
|
|
"eval_samples_per_second": 0.334,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2720633002475918,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000231427255935,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7819440018314133,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27638481627584127,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021283950173925044,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.6,
|
|
"completions/max_terminated_length": 261.6,
|
|
"completions/mean_length": 193.0875,
|
|
"completions/mean_terminated_length": 193.0875,
|
|
"completions/min_length": 135.9,
|
|
"completions/min_terminated_length": 135.9,
|
|
"entropy": 1.1175101712346076,
|
|
"epoch": 0.9202905212834421,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.9315080642700195,
|
|
"kl": 0.04182431444060057,
|
|
"learning_rate": 4.151897551524562e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 43526478.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.2504029482603073,
|
|
"sampling/importance_sampling_ratio/max": 1.279539656639099,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005105555057525,
|
|
"sampling/importance_sampling_ratio/min": 0.7738273978233338,
|
|
"sampling/sampling_logp_difference/max": 0.300219988822937,
|
|
"sampling/sampling_logp_difference/mean": 0.022065847553312777,
|
|
"step": 10010
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.7,
|
|
"completions/max_terminated_length": 303.7,
|
|
"completions/mean_length": 218.3625,
|
|
"completions/mean_terminated_length": 218.3625,
|
|
"completions/min_length": 147.8,
|
|
"completions/min_terminated_length": 147.8,
|
|
"entropy": 1.078858845680952,
|
|
"epoch": 0.9212098924335754,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 7.221519470214844,
|
|
"kl": 0.034373096813214946,
|
|
"learning_rate": 4.149994970014461e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 43570179.0,
|
|
"reward": -0.075,
|
|
"reward_std": 0.49618045389652254,
|
|
"rewards/reward_fn/mean": -0.075,
|
|
"rewards/reward_fn/std": 0.496180459856987,
|
|
"sampling/importance_sampling_ratio/max": 1.4886763453483582,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004341721534729,
|
|
"sampling/importance_sampling_ratio/min": 0.7414919972419739,
|
|
"sampling/sampling_logp_difference/max": 0.447513222694397,
|
|
"sampling/sampling_logp_difference/mean": 0.021396753191947938,
|
|
"step": 10020
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 271.0,
|
|
"completions/max_terminated_length": 271.0,
|
|
"completions/mean_length": 199.425,
|
|
"completions/mean_terminated_length": 199.425,
|
|
"completions/min_length": 129.1,
|
|
"completions/min_terminated_length": 129.1,
|
|
"entropy": 1.1138561457395553,
|
|
"epoch": 0.9221292635837087,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.212597846984863,
|
|
"kl": 0.05524435707484372,
|
|
"learning_rate": 4.1480906938272625e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 43611853.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.3167575001716614,
|
|
"rewards/reward_fn/mean": 0.375,
|
|
"rewards/reward_fn/std": 0.3167575001716614,
|
|
"sampling/importance_sampling_ratio/max": 1.2674006819725037,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998987436294555,
|
|
"sampling/importance_sampling_ratio/min": 0.7750279426574707,
|
|
"sampling/sampling_logp_difference/max": 0.2715704143047333,
|
|
"sampling/sampling_logp_difference/mean": 0.022041104920208453,
|
|
"step": 10030
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.3,
|
|
"completions/max_terminated_length": 316.3,
|
|
"completions/mean_length": 218.375,
|
|
"completions/mean_terminated_length": 218.375,
|
|
"completions/min_length": 143.2,
|
|
"completions/min_terminated_length": 143.2,
|
|
"entropy": 1.1344326540827752,
|
|
"epoch": 0.923048634733842,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.12102018296718597,
|
|
"kl": 0.04625592551892623,
|
|
"learning_rate": 4.14618472491881e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 43656387.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.13509859144687653,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.13509859144687653,
|
|
"sampling/importance_sampling_ratio/max": 1.2803356528282166,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999973893165588,
|
|
"sampling/importance_sampling_ratio/min": 0.7905018031597137,
|
|
"sampling/sampling_logp_difference/max": 0.2589735984802246,
|
|
"sampling/sampling_logp_difference/mean": 0.021617263741791248,
|
|
"step": 10040
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.4,
|
|
"completions/max_terminated_length": 276.4,
|
|
"completions/mean_length": 202.1375,
|
|
"completions/mean_terminated_length": 202.1375,
|
|
"completions/min_length": 133.3,
|
|
"completions/min_terminated_length": 133.3,
|
|
"entropy": 1.023564712703228,
|
|
"epoch": 0.9239680058839753,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.862313747406006,
|
|
"kl": 0.043661712884204465,
|
|
"learning_rate": 4.1442770652466894e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 43697830.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.34637176990509033,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.3463717758655548,
|
|
"sampling/importance_sampling_ratio/max": 1.2744048953056335,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996317446231842,
|
|
"sampling/importance_sampling_ratio/min": 0.7942969143390656,
|
|
"sampling/sampling_logp_difference/max": 0.24789116382598878,
|
|
"sampling/sampling_logp_difference/mean": 0.020310519449412822,
|
|
"step": 10050
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 321.9,
|
|
"completions/max_terminated_length": 304.3,
|
|
"completions/mean_length": 214.0,
|
|
"completions/mean_terminated_length": 210.37857208251953,
|
|
"completions/min_length": 134.7,
|
|
"completions/min_terminated_length": 134.7,
|
|
"entropy": 1.159387330710888,
|
|
"epoch": 0.9248873770341086,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.11104026436805725,
|
|
"kl": 0.04339903194922954,
|
|
"learning_rate": 4.142367716770216e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 43741574.0,
|
|
"reward": 0.11875,
|
|
"reward_std": 0.37845076620578766,
|
|
"rewards/reward_fn/mean": 0.11875,
|
|
"rewards/reward_fn/std": 0.3784507840871811,
|
|
"sampling/importance_sampling_ratio/max": 1.271242046356201,
|
|
"sampling/importance_sampling_ratio/mean": 1.000460785627365,
|
|
"sampling/importance_sampling_ratio/min": 0.750554895401001,
|
|
"sampling/sampling_logp_difference/max": 0.31674994230270387,
|
|
"sampling/sampling_logp_difference/mean": 0.023078887723386288,
|
|
"step": 10060
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.9,
|
|
"completions/max_terminated_length": 326.9,
|
|
"completions/mean_length": 219.0375,
|
|
"completions/mean_terminated_length": 219.0375,
|
|
"completions/min_length": 133.6,
|
|
"completions/min_terminated_length": 133.6,
|
|
"entropy": 1.1225076988339424,
|
|
"epoch": 0.925806748184242,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.00573205947876,
|
|
"kl": 0.03885363736189902,
|
|
"learning_rate": 4.1404566814504465e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 43785905.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.34120489954948424,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.34120491743087766,
|
|
"sampling/importance_sampling_ratio/max": 1.3655216336250304,
|
|
"sampling/importance_sampling_ratio/mean": 1.0005740821361542,
|
|
"sampling/importance_sampling_ratio/min": 0.7938507914543151,
|
|
"sampling/sampling_logp_difference/max": 0.319132924079895,
|
|
"sampling/sampling_logp_difference/mean": 0.021599754877388477,
|
|
"step": 10070
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.9,
|
|
"completions/max_terminated_length": 311.9,
|
|
"completions/mean_length": 214.0,
|
|
"completions/mean_terminated_length": 214.0,
|
|
"completions/min_length": 136.2,
|
|
"completions/min_terminated_length": 136.2,
|
|
"entropy": 1.0852583475410937,
|
|
"epoch": 0.9267261193343753,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.248187065124512,
|
|
"kl": 0.043491474015172574,
|
|
"learning_rate": 4.138543961250167e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 43830665.0,
|
|
"reward": -0.2,
|
|
"reward_std": 0.34928734302520753,
|
|
"rewards/reward_fn/mean": -0.2,
|
|
"rewards/reward_fn/std": 0.3492873549461365,
|
|
"sampling/importance_sampling_ratio/max": 1.261192524433136,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002699196338654,
|
|
"sampling/importance_sampling_ratio/min": 0.783576500415802,
|
|
"sampling/sampling_logp_difference/max": 0.2687194228172302,
|
|
"sampling/sampling_logp_difference/mean": 0.021574455499649047,
|
|
"step": 10080
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.6,
|
|
"completions/max_terminated_length": 290.6,
|
|
"completions/mean_length": 198.5125,
|
|
"completions/mean_terminated_length": 198.5125,
|
|
"completions/min_length": 125.0,
|
|
"completions/min_terminated_length": 125.0,
|
|
"entropy": 1.0963963769376277,
|
|
"epoch": 0.9276454904845086,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.16739299893379211,
|
|
"kl": 0.04293804967892356,
|
|
"learning_rate": 4.136629558133892e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 43875738.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.30754689276218417,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.3075469046831131,
|
|
"sampling/importance_sampling_ratio/max": 1.2990479350090027,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999501526355743,
|
|
"sampling/importance_sampling_ratio/min": 0.7951829969882965,
|
|
"sampling/sampling_logp_difference/max": 0.2698585748672485,
|
|
"sampling/sampling_logp_difference/mean": 0.02218684498220682,
|
|
"step": 10090
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.6,
|
|
"completions/max_terminated_length": 269.6,
|
|
"completions/mean_length": 198.025,
|
|
"completions/mean_terminated_length": 198.025,
|
|
"completions/min_length": 127.6,
|
|
"completions/min_terminated_length": 127.6,
|
|
"entropy": 1.0207598965615035,
|
|
"epoch": 0.9285648616346419,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.476590633392334,
|
|
"kl": 0.04273106640903279,
|
|
"learning_rate": 4.134713474067871e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 43919132.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.3834284096956253,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.38342841565608976,
|
|
"sampling/importance_sampling_ratio/max": 1.241877281665802,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000837743282318,
|
|
"sampling/importance_sampling_ratio/min": 0.7797056853771209,
|
|
"sampling/sampling_logp_difference/max": 0.26245195865631105,
|
|
"sampling/sampling_logp_difference/mean": 0.020670629851520062,
|
|
"step": 10100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 322.4,
|
|
"completions/max_terminated_length": 322.4,
|
|
"completions/mean_length": 227.55,
|
|
"completions/mean_terminated_length": 227.55,
|
|
"completions/min_length": 152.8,
|
|
"completions/min_terminated_length": 152.8,
|
|
"entropy": 1.0423229604959487,
|
|
"epoch": 0.9294842327847752,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.10241851210594177,
|
|
"kl": 0.033168822061270475,
|
|
"learning_rate": 4.132795711020072e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 43964264.0,
|
|
"reward": -0.1625,
|
|
"reward_std": 0.3882720351219177,
|
|
"rewards/reward_fn/mean": -0.1625,
|
|
"rewards/reward_fn/std": 0.38827205300331114,
|
|
"sampling/importance_sampling_ratio/max": 1.2814345121383668,
|
|
"sampling/importance_sampling_ratio/mean": 1.000113022327423,
|
|
"sampling/importance_sampling_ratio/min": 0.7712997794151306,
|
|
"sampling/sampling_logp_difference/max": 0.2917073488235474,
|
|
"sampling/sampling_logp_difference/mean": 0.020850983448326588,
|
|
"step": 10110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.8,
|
|
"completions/max_terminated_length": 272.8,
|
|
"completions/mean_length": 193.2625,
|
|
"completions/mean_terminated_length": 193.2625,
|
|
"completions/min_length": 123.2,
|
|
"completions/min_terminated_length": 123.2,
|
|
"entropy": 1.0889507330954076,
|
|
"epoch": 0.9304036039349085,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.15005534887313843,
|
|
"kl": 0.03510474353097379,
|
|
"learning_rate": 4.130876270960194e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 44006533.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.11700168251991272,
|
|
"rewards/reward_fn/mean": 0.375,
|
|
"rewards/reward_fn/std": 0.11700168251991272,
|
|
"sampling/importance_sampling_ratio/max": 1.242002785205841,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996794700622559,
|
|
"sampling/importance_sampling_ratio/min": 0.7888605296611786,
|
|
"sampling/sampling_logp_difference/max": 0.24492315649986268,
|
|
"sampling/sampling_logp_difference/mean": 0.02183618824928999,
|
|
"step": 10120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 257.0,
|
|
"completions/max_terminated_length": 257.0,
|
|
"completions/mean_length": 186.45,
|
|
"completions/mean_terminated_length": 186.45,
|
|
"completions/min_length": 133.7,
|
|
"completions/min_terminated_length": 133.7,
|
|
"entropy": 1.0243648651987314,
|
|
"epoch": 0.9313229750850418,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.017781734466553,
|
|
"kl": 0.04347193767898716,
|
|
"learning_rate": 4.128955155859655e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 44048841.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.23830750584602356,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.23830751776695253,
|
|
"sampling/importance_sampling_ratio/max": 1.2669746160507203,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998945713043212,
|
|
"sampling/importance_sampling_ratio/min": 0.800959974527359,
|
|
"sampling/sampling_logp_difference/max": 0.255924391746521,
|
|
"sampling/sampling_logp_difference/mean": 0.020703633781522512,
|
|
"step": 10130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.6,
|
|
"completions/max_terminated_length": 298.6,
|
|
"completions/mean_length": 209.8375,
|
|
"completions/mean_terminated_length": 209.8375,
|
|
"completions/min_length": 130.9,
|
|
"completions/min_terminated_length": 130.9,
|
|
"entropy": 1.0491491258144379,
|
|
"epoch": 0.9322423462351751,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.495232582092285,
|
|
"kl": 0.03814993262058124,
|
|
"learning_rate": 4.127032367691595e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 44094612.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.3202547788619995,
|
|
"sampling/importance_sampling_ratio/max": 1.2811823964118958,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000887930393219,
|
|
"sampling/importance_sampling_ratio/min": 0.7703660070896149,
|
|
"sampling/sampling_logp_difference/max": 0.2962468147277832,
|
|
"sampling/sampling_logp_difference/mean": 0.02153892759233713,
|
|
"step": 10140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.3,
|
|
"completions/max_terminated_length": 280.3,
|
|
"completions/mean_length": 197.25,
|
|
"completions/mean_terminated_length": 197.25,
|
|
"completions/min_length": 131.7,
|
|
"completions/min_terminated_length": 131.7,
|
|
"entropy": 1.121644602715969,
|
|
"epoch": 0.9331617173853084,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.980490684509277,
|
|
"kl": 0.0493826134246774,
|
|
"learning_rate": 4.1251079084308696e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 44134688.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.2722664296627045,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.2722664475440979,
|
|
"sampling/importance_sampling_ratio/max": 1.2625227212905883,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004570841789246,
|
|
"sampling/importance_sampling_ratio/min": 0.8016109704971314,
|
|
"sampling/sampling_logp_difference/max": 0.2479767084121704,
|
|
"sampling/sampling_logp_difference/mean": 0.022446214407682418,
|
|
"step": 10150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.0,
|
|
"completions/max_terminated_length": 310.0,
|
|
"completions/mean_length": 219.35,
|
|
"completions/mean_terminated_length": 219.35,
|
|
"completions/min_length": 146.6,
|
|
"completions/min_terminated_length": 146.6,
|
|
"entropy": 1.1479847468435764,
|
|
"epoch": 0.9340810885354418,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.437140464782715,
|
|
"kl": 0.038918071117950606,
|
|
"learning_rate": 4.123181780054054e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 44178548.0,
|
|
"reward": -0.2625,
|
|
"reward_std": 0.3743988037109375,
|
|
"rewards/reward_fn/mean": -0.2625,
|
|
"rewards/reward_fn/std": 0.37439881563186644,
|
|
"sampling/importance_sampling_ratio/max": 1.2357716679573059,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000833213329314,
|
|
"sampling/importance_sampling_ratio/min": 0.7927090287208557,
|
|
"sampling/sampling_logp_difference/max": 0.24702129364013672,
|
|
"sampling/sampling_logp_difference/mean": 0.02197853084653616,
|
|
"step": 10160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 310.7,
|
|
"completions/max_terminated_length": 310.7,
|
|
"completions/mean_length": 204.675,
|
|
"completions/mean_terminated_length": 204.675,
|
|
"completions/min_length": 122.5,
|
|
"completions/min_terminated_length": 122.5,
|
|
"entropy": 1.1130825310945511,
|
|
"epoch": 0.9350004596855751,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.14885349571704865,
|
|
"kl": 0.03417475215392187,
|
|
"learning_rate": 4.121253984539437e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 44222338.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.336654344201088,
|
|
"rewards/reward_fn/mean": 0.225,
|
|
"rewards/reward_fn/std": 0.3366543620824814,
|
|
"sampling/importance_sampling_ratio/max": 1.2490541458129882,
|
|
"sampling/importance_sampling_ratio/mean": 0.999740082025528,
|
|
"sampling/importance_sampling_ratio/min": 0.7749578654766083,
|
|
"sampling/sampling_logp_difference/max": 0.26064772605895997,
|
|
"sampling/sampling_logp_difference/mean": 0.021991936862468718,
|
|
"step": 10170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.7,
|
|
"completions/max_terminated_length": 279.7,
|
|
"completions/mean_length": 194.7125,
|
|
"completions/mean_terminated_length": 194.7125,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 1.1041462056338787,
|
|
"epoch": 0.9359198308357084,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.3849334716796875,
|
|
"kl": 0.03447888652444817,
|
|
"learning_rate": 4.1193245238670185e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 44263099.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.2581017464399338,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.25810175240039823,
|
|
"sampling/importance_sampling_ratio/max": 1.293313467502594,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996035099029541,
|
|
"sampling/importance_sampling_ratio/min": 0.7614112794399261,
|
|
"sampling/sampling_logp_difference/max": 0.288004732131958,
|
|
"sampling/sampling_logp_difference/mean": 0.02223816458135843,
|
|
"step": 10180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 309.1,
|
|
"completions/max_terminated_length": 309.1,
|
|
"completions/mean_length": 227.825,
|
|
"completions/mean_terminated_length": 227.825,
|
|
"completions/min_length": 161.6,
|
|
"completions/min_terminated_length": 161.6,
|
|
"entropy": 1.059245329350233,
|
|
"epoch": 0.9368392019858417,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.509138107299805,
|
|
"kl": 0.042996865278109905,
|
|
"learning_rate": 4.117393400018508e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 44307525.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.5376395136117935,
|
|
"rewards/reward_fn/mean": 0.275,
|
|
"rewards/reward_fn/std": 0.537639531493187,
|
|
"sampling/importance_sampling_ratio/max": 1.243626093864441,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997692883014679,
|
|
"sampling/importance_sampling_ratio/min": 0.771395879983902,
|
|
"sampling/sampling_logp_difference/max": 0.2736286222934723,
|
|
"sampling/sampling_logp_difference/mean": 0.021303676813840867,
|
|
"step": 10190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.6,
|
|
"completions/max_terminated_length": 261.6,
|
|
"completions/mean_length": 192.8125,
|
|
"completions/mean_terminated_length": 192.8125,
|
|
"completions/min_length": 136.7,
|
|
"completions/min_terminated_length": 136.7,
|
|
"entropy": 1.1562938824295999,
|
|
"epoch": 0.937758573135975,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.12893037497997284,
|
|
"kl": 0.03760570762678981,
|
|
"learning_rate": 4.115460614977327e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 44351230.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.32165118157863615,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.32165118753910066,
|
|
"sampling/importance_sampling_ratio/max": 1.2510242104530334,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993673980236053,
|
|
"sampling/importance_sampling_ratio/min": 0.8053878366947174,
|
|
"sampling/sampling_logp_difference/max": 0.24389951229095458,
|
|
"sampling/sampling_logp_difference/mean": 0.02238678764551878,
|
|
"step": 10200
|
|
},
|
|
{
|
|
"epoch": 0.937758573135975,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 300.8848167539267,
|
|
"eval_completions/max_terminated_length": 299.67539267015707,
|
|
"eval_completions/mean_length": 212.23276614310646,
|
|
"eval_completions/mean_terminated_length": 211.8950490635311,
|
|
"eval_completions/min_length": 138.26352530541013,
|
|
"eval_completions/min_terminated_length": 138.26352530541013,
|
|
"eval_entropy": 1.1049813820637542,
|
|
"eval_frac_reward_zero_std": 0.3298429319371728,
|
|
"eval_kl": 0.03762181090226764,
|
|
"eval_loss": 0.0018853785004466772,
|
|
"eval_num_tokens": 44351230.0,
|
|
"eval_reward": 0.06479057591623037,
|
|
"eval_reward_std": 0.3453926005600635,
|
|
"eval_rewards/reward_fn/mean": 0.06479057591623037,
|
|
"eval_rewards/reward_fn/std": 0.3453926114823806,
|
|
"eval_runtime": 1753.6436,
|
|
"eval_samples_per_second": 0.327,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2748777233909354,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000669922920213,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7792097128290573,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2789977702586855,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021610155227036583,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 10200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 345.5,
|
|
"completions/max_terminated_length": 321.4,
|
|
"completions/mean_length": 237.5625,
|
|
"completions/mean_terminated_length": 233.4232147216797,
|
|
"completions/min_length": 149.8,
|
|
"completions/min_terminated_length": 149.8,
|
|
"entropy": 1.1053246222436428,
|
|
"epoch": 0.9386779442861083,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.814712047576904,
|
|
"kl": 0.0349264758056961,
|
|
"learning_rate": 4.113526170728599e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 44393211.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.41637334525585173,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.41637335121631625,
|
|
"sampling/importance_sampling_ratio/max": 1.3118617177009582,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999534308910369,
|
|
"sampling/importance_sampling_ratio/min": 0.7669457614421844,
|
|
"sampling/sampling_logp_difference/max": 0.30400129556655886,
|
|
"sampling/sampling_logp_difference/mean": 0.022214807011187075,
|
|
"step": 10210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.0,
|
|
"completions/max_terminated_length": 293.0,
|
|
"completions/mean_length": 208.4375,
|
|
"completions/mean_terminated_length": 208.4375,
|
|
"completions/min_length": 148.5,
|
|
"completions/min_terminated_length": 148.5,
|
|
"entropy": 1.1892416171729565,
|
|
"epoch": 0.9395973154362416,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.578960418701172,
|
|
"kl": 0.05054844588739797,
|
|
"learning_rate": 4.111590069259154e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 44440254.0,
|
|
"reward": -0.3375,
|
|
"reward_std": 0.3981345325708389,
|
|
"rewards/reward_fn/mean": -0.3375,
|
|
"rewards/reward_fn/std": 0.3981345325708389,
|
|
"sampling/importance_sampling_ratio/max": 1.265317153930664,
|
|
"sampling/importance_sampling_ratio/mean": 0.9993602216243744,
|
|
"sampling/importance_sampling_ratio/min": 0.8003243505954742,
|
|
"sampling/sampling_logp_difference/max": 0.257047438621521,
|
|
"sampling/sampling_logp_difference/mean": 0.02259406503289938,
|
|
"step": 10220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.3,
|
|
"completions/max_terminated_length": 293.3,
|
|
"completions/mean_length": 218.675,
|
|
"completions/mean_terminated_length": 218.675,
|
|
"completions/min_length": 146.6,
|
|
"completions/min_terminated_length": 146.6,
|
|
"entropy": 1.0946271598339081,
|
|
"epoch": 0.9405166865863749,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.726904392242432,
|
|
"kl": 0.03711949006537907,
|
|
"learning_rate": 4.109652312557523e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 44484620.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/reward_fn/mean": 0.425,
|
|
"rewards/reward_fn/std": 0.3202547788619995,
|
|
"sampling/importance_sampling_ratio/max": 1.2811763882637024,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998382806777955,
|
|
"sampling/importance_sampling_ratio/min": 0.800939393043518,
|
|
"sampling/sampling_logp_difference/max": 0.25599147081375123,
|
|
"sampling/sampling_logp_difference/mean": 0.021071030013263225,
|
|
"step": 10230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.0,
|
|
"completions/max_terminated_length": 291.0,
|
|
"completions/mean_length": 209.725,
|
|
"completions/mean_terminated_length": 209.725,
|
|
"completions/min_length": 136.7,
|
|
"completions/min_terminated_length": 136.7,
|
|
"entropy": 1.0350177951157093,
|
|
"epoch": 0.9414360577365082,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.088034629821777,
|
|
"kl": 0.024944693577708676,
|
|
"learning_rate": 4.107712902613936e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 44527414.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.3484488636255264,
|
|
"rewards/reward_fn/mean": 0.5625,
|
|
"rewards/reward_fn/std": 0.34844887554645537,
|
|
"sampling/importance_sampling_ratio/max": 1.3179100394248962,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999079942703247,
|
|
"sampling/importance_sampling_ratio/min": 0.731504213809967,
|
|
"sampling/sampling_logp_difference/max": 0.35937021374702455,
|
|
"sampling/sampling_logp_difference/mean": 0.02133547030389309,
|
|
"step": 10240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 301.2,
|
|
"completions/max_terminated_length": 301.2,
|
|
"completions/mean_length": 212.9,
|
|
"completions/mean_terminated_length": 212.9,
|
|
"completions/min_length": 139.6,
|
|
"completions/min_terminated_length": 139.6,
|
|
"entropy": 1.0743322379887104,
|
|
"epoch": 0.9423554288866416,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.15123139321804047,
|
|
"kl": 0.04109329528873786,
|
|
"learning_rate": 4.105771841420326e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 44571198.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.34728909432888033,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.34728911221027375,
|
|
"sampling/importance_sampling_ratio/max": 1.3009016036987304,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996633112430573,
|
|
"sampling/importance_sampling_ratio/min": 0.7681370854377747,
|
|
"sampling/sampling_logp_difference/max": 0.2990386962890625,
|
|
"sampling/sampling_logp_difference/mean": 0.020904244296252726,
|
|
"step": 10250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.1,
|
|
"completions/max_terminated_length": 277.1,
|
|
"completions/mean_length": 198.0375,
|
|
"completions/mean_terminated_length": 198.0375,
|
|
"completions/min_length": 127.8,
|
|
"completions/min_terminated_length": 127.8,
|
|
"entropy": 1.1008296683430672,
|
|
"epoch": 0.9432748000367749,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.316898822784424,
|
|
"kl": 0.052229932916816324,
|
|
"learning_rate": 4.103829130970316e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 44613385.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.30008076429367064,
|
|
"rewards/reward_fn/mean": 0.0375,
|
|
"rewards/reward_fn/std": 0.30008077025413515,
|
|
"sampling/importance_sampling_ratio/max": 1.3893833875656127,
|
|
"sampling/importance_sampling_ratio/mean": 1.000041651725769,
|
|
"sampling/importance_sampling_ratio/min": 0.7663739085197449,
|
|
"sampling/sampling_logp_difference/max": 0.38043294548988343,
|
|
"sampling/sampling_logp_difference/mean": 0.021824513375759125,
|
|
"step": 10260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 298.8,
|
|
"completions/max_terminated_length": 298.8,
|
|
"completions/mean_length": 201.95,
|
|
"completions/mean_terminated_length": 201.95,
|
|
"completions/min_length": 128.7,
|
|
"completions/min_terminated_length": 128.7,
|
|
"entropy": 1.087373359501362,
|
|
"epoch": 0.9441941711869082,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.971785068511963,
|
|
"kl": 0.03778417706489563,
|
|
"learning_rate": 4.101884773259226e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 44655949.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.36277919411659243,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.36277919411659243,
|
|
"sampling/importance_sampling_ratio/max": 1.2491565942764282,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001887202262878,
|
|
"sampling/importance_sampling_ratio/min": 0.7989834904670715,
|
|
"sampling/sampling_logp_difference/max": 0.24308784008026124,
|
|
"sampling/sampling_logp_difference/mean": 0.021357456222176552,
|
|
"step": 10270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.7,
|
|
"completions/max_terminated_length": 308.7,
|
|
"completions/mean_length": 219.4375,
|
|
"completions/mean_terminated_length": 219.4375,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"entropy": 1.1221811167895794,
|
|
"epoch": 0.9451135423370415,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.410228729248047,
|
|
"kl": 0.0418703637260478,
|
|
"learning_rate": 4.099938770284066e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 44702456.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.42824027836322787,
|
|
"rewards/reward_fn/mean": 0.3,
|
|
"rewards/reward_fn/std": 0.4282402962446213,
|
|
"sampling/importance_sampling_ratio/max": 1.3488599181175231,
|
|
"sampling/importance_sampling_ratio/mean": 1.000151389837265,
|
|
"sampling/importance_sampling_ratio/min": 0.7894112348556519,
|
|
"sampling/sampling_logp_difference/max": 0.29748125076293946,
|
|
"sampling/sampling_logp_difference/mean": 0.021881341747939587,
|
|
"step": 10280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 288.4,
|
|
"completions/max_terminated_length": 288.4,
|
|
"completions/mean_length": 210.875,
|
|
"completions/mean_terminated_length": 210.875,
|
|
"completions/min_length": 141.5,
|
|
"completions/min_terminated_length": 141.5,
|
|
"entropy": 1.1194141037762164,
|
|
"epoch": 0.9460329134871748,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.95265007019043,
|
|
"kl": 0.03852440757327713,
|
|
"learning_rate": 4.09799112404354e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 44746390.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.22768060266971588,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.22768060266971588,
|
|
"sampling/importance_sampling_ratio/max": 1.2550797700881957,
|
|
"sampling/importance_sampling_ratio/mean": 1.0006340980529784,
|
|
"sampling/importance_sampling_ratio/min": 0.7692474305629731,
|
|
"sampling/sampling_logp_difference/max": 0.2870879292488098,
|
|
"sampling/sampling_logp_difference/mean": 0.02200090270489454,
|
|
"step": 10290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.4,
|
|
"completions/max_terminated_length": 312.4,
|
|
"completions/mean_length": 221.6875,
|
|
"completions/mean_terminated_length": 221.6875,
|
|
"completions/min_length": 152.8,
|
|
"completions/min_terminated_length": 152.8,
|
|
"entropy": 1.1098796352744102,
|
|
"epoch": 0.9469522846373081,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.12116257101297379,
|
|
"kl": 0.04647766302805394,
|
|
"learning_rate": 4.096041836538034e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 44795693.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.2205115258693695,
|
|
"sampling/importance_sampling_ratio/max": 1.2870426893234252,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997973620891571,
|
|
"sampling/importance_sampling_ratio/min": 0.7948153734207153,
|
|
"sampling/sampling_logp_difference/max": 0.26373398303985596,
|
|
"sampling/sampling_logp_difference/mean": 0.021479869447648525,
|
|
"step": 10300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 344.1,
|
|
"completions/max_terminated_length": 344.1,
|
|
"completions/mean_length": 231.1,
|
|
"completions/mean_terminated_length": 231.1,
|
|
"completions/min_length": 140.3,
|
|
"completions/min_terminated_length": 140.3,
|
|
"entropy": 1.074521803855896,
|
|
"epoch": 0.9478716557874414,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 4.28732967376709,
|
|
"kl": 0.031806351919658485,
|
|
"learning_rate": 4.094090909769624e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 44841069.0,
|
|
"reward": 0.11875,
|
|
"reward_std": 0.2901452392339706,
|
|
"rewards/reward_fn/mean": 0.11875,
|
|
"rewards/reward_fn/std": 0.2901452511548996,
|
|
"sampling/importance_sampling_ratio/max": 1.2504416108131409,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999383211135864,
|
|
"sampling/importance_sampling_ratio/min": 0.8030679225921631,
|
|
"sampling/sampling_logp_difference/max": 0.23733532428741455,
|
|
"sampling/sampling_logp_difference/mean": 0.021142423897981644,
|
|
"step": 10310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.2,
|
|
"completions/max_terminated_length": 291.2,
|
|
"completions/mean_length": 212.4875,
|
|
"completions/mean_terminated_length": 212.4875,
|
|
"completions/min_length": 137.7,
|
|
"completions/min_terminated_length": 137.7,
|
|
"entropy": 1.1351409830152988,
|
|
"epoch": 0.9487910269375747,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.122869491577148,
|
|
"kl": 0.04445941095473245,
|
|
"learning_rate": 4.092138345742066e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 44884084.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.43609668016433717,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.4360966980457306,
|
|
"sampling/importance_sampling_ratio/max": 1.2811456561088561,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999609231948853,
|
|
"sampling/importance_sampling_ratio/min": 0.8035965740680695,
|
|
"sampling/sampling_logp_difference/max": 0.26156005859375,
|
|
"sampling/sampling_logp_difference/mean": 0.022171403281390667,
|
|
"step": 10320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.8,
|
|
"completions/max_terminated_length": 289.8,
|
|
"completions/mean_length": 200.4625,
|
|
"completions/mean_terminated_length": 200.4625,
|
|
"completions/min_length": 123.5,
|
|
"completions/min_terminated_length": 123.5,
|
|
"entropy": 1.09742796048522,
|
|
"epoch": 0.949710398087708,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 7.2112860679626465,
|
|
"kl": 0.04795564757660031,
|
|
"learning_rate": 4.090184146460803e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 44924873.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.43528797626495364,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.43528798818588255,
|
|
"sampling/importance_sampling_ratio/max": 1.2881731629371642,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997185647487641,
|
|
"sampling/importance_sampling_ratio/min": 0.768883228302002,
|
|
"sampling/sampling_logp_difference/max": 0.30654571056365965,
|
|
"sampling/sampling_logp_difference/mean": 0.021639818139374257,
|
|
"step": 10330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.1,
|
|
"completions/max_terminated_length": 286.1,
|
|
"completions/mean_length": 209.4375,
|
|
"completions/mean_terminated_length": 209.4375,
|
|
"completions/min_length": 147.5,
|
|
"completions/min_terminated_length": 147.5,
|
|
"entropy": 1.1303051091730594,
|
|
"epoch": 0.9506297692378414,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.516763687133789,
|
|
"kl": 0.04249354059575126,
|
|
"learning_rate": 4.088228313932952e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 44973852.0,
|
|
"reward": -0.175,
|
|
"reward_std": 0.48268860280513765,
|
|
"rewards/reward_fn/mean": -0.175,
|
|
"rewards/reward_fn/std": 0.48268862068653107,
|
|
"sampling/importance_sampling_ratio/max": 1.2844378232955933,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999155104160309,
|
|
"sampling/importance_sampling_ratio/min": 0.7481306195259094,
|
|
"sampling/sampling_logp_difference/max": 0.3106665849685669,
|
|
"sampling/sampling_logp_difference/mean": 0.0215362723916769,
|
|
"step": 10340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 324.5,
|
|
"completions/max_terminated_length": 314.3,
|
|
"completions/mean_length": 223.8875,
|
|
"completions/mean_terminated_length": 220.54107208251952,
|
|
"completions/min_length": 133.9,
|
|
"completions/min_terminated_length": 133.9,
|
|
"entropy": 1.0112129610031844,
|
|
"epoch": 0.9515491403879747,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.894485950469971,
|
|
"kl": 0.03037277526455,
|
|
"learning_rate": 4.08627085016731e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 45019083.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.479353791475296,
|
|
"rewards/reward_fn/mean": 0.09375,
|
|
"rewards/reward_fn/std": 0.4793538093566895,
|
|
"sampling/importance_sampling_ratio/max": 1.2665707349777222,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999996721744537,
|
|
"sampling/importance_sampling_ratio/min": 0.7669129192829132,
|
|
"sampling/sampling_logp_difference/max": 0.27528184056282046,
|
|
"sampling/sampling_logp_difference/mean": 0.020327751338481904,
|
|
"step": 10350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.0,
|
|
"completions/max_terminated_length": 306.0,
|
|
"completions/mean_length": 224.5,
|
|
"completions/mean_terminated_length": 224.5,
|
|
"completions/min_length": 147.3,
|
|
"completions/min_terminated_length": 147.3,
|
|
"entropy": 1.100806525349617,
|
|
"epoch": 0.952468511538108,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.407718181610107,
|
|
"kl": 0.03967062687734142,
|
|
"learning_rate": 4.084311757174351e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 45064171.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.348606663942337,
|
|
"rewards/reward_fn/mean": 0.2625,
|
|
"rewards/reward_fn/std": 0.348606663942337,
|
|
"sampling/importance_sampling_ratio/max": 1.2466299176216125,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002277851104737,
|
|
"sampling/importance_sampling_ratio/min": 0.79264235496521,
|
|
"sampling/sampling_logp_difference/max": 0.2533001065254211,
|
|
"sampling/sampling_logp_difference/mean": 0.02188583854585886,
|
|
"step": 10360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 336.1,
|
|
"completions/max_terminated_length": 322.4,
|
|
"completions/mean_length": 232.3125,
|
|
"completions/mean_terminated_length": 229.16607360839845,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 1.0800414830446243,
|
|
"epoch": 0.9533878826882413,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.4677653312683105,
|
|
"kl": 0.03788230170030147,
|
|
"learning_rate": 4.082351036966219e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 45113820.0,
|
|
"reward": -0.03125,
|
|
"reward_std": 0.3796415150165558,
|
|
"rewards/reward_fn/mean": -0.03125,
|
|
"rewards/reward_fn/std": 0.37964152693748476,
|
|
"sampling/importance_sampling_ratio/max": 1.3185320973396302,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000298917293549,
|
|
"sampling/importance_sampling_ratio/min": 0.7852094233036041,
|
|
"sampling/sampling_logp_difference/max": 0.29634692668914797,
|
|
"sampling/sampling_logp_difference/mean": 0.02104109898209572,
|
|
"step": 10370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.4,
|
|
"completions/max_terminated_length": 295.4,
|
|
"completions/mean_length": 210.1625,
|
|
"completions/mean_terminated_length": 210.1625,
|
|
"completions/min_length": 132.5,
|
|
"completions/min_terminated_length": 132.5,
|
|
"entropy": 1.134573294967413,
|
|
"epoch": 0.9543072538383746,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.526520252227783,
|
|
"kl": 0.04285432985052466,
|
|
"learning_rate": 4.080388691556731e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 45158217.0,
|
|
"reward": -0.05,
|
|
"reward_std": 0.18685350716114044,
|
|
"rewards/reward_fn/mean": -0.05,
|
|
"rewards/reward_fn/std": 0.18685351312160492,
|
|
"sampling/importance_sampling_ratio/max": 1.3135084986686707,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004297852516175,
|
|
"sampling/importance_sampling_ratio/min": 0.7943588852882385,
|
|
"sampling/sampling_logp_difference/max": 0.2781773805618286,
|
|
"sampling/sampling_logp_difference/mean": 0.021459191851317883,
|
|
"step": 10380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 292.1,
|
|
"completions/max_terminated_length": 292.1,
|
|
"completions/mean_length": 223.15,
|
|
"completions/mean_terminated_length": 223.15,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 0.9993706054985523,
|
|
"epoch": 0.9552266249885079,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.806354522705078,
|
|
"kl": 0.05351685233181343,
|
|
"learning_rate": 4.0784247229613725e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 45203213.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.5401845872402191,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.5401846051216126,
|
|
"sampling/importance_sampling_ratio/max": 1.3444734692573548,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998006582260132,
|
|
"sampling/importance_sampling_ratio/min": 0.7332471027970314,
|
|
"sampling/sampling_logp_difference/max": 0.42358884811401365,
|
|
"sampling/sampling_logp_difference/mean": 0.02031092420220375,
|
|
"step": 10390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 337.3,
|
|
"completions/max_terminated_length": 337.3,
|
|
"completions/mean_length": 228.9875,
|
|
"completions/mean_terminated_length": 228.9875,
|
|
"completions/min_length": 141.3,
|
|
"completions/min_terminated_length": 141.3,
|
|
"entropy": 1.092549277842045,
|
|
"epoch": 0.9561459961386412,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.153485506772995,
|
|
"kl": 0.04453683134051971,
|
|
"learning_rate": 4.076459133197297e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 45250036.0,
|
|
"reward": -0.1375,
|
|
"reward_std": 0.22768060266971588,
|
|
"rewards/reward_fn/mean": -0.1375,
|
|
"rewards/reward_fn/std": 0.22768060266971588,
|
|
"sampling/importance_sampling_ratio/max": 1.2927642822265626,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999365508556366,
|
|
"sampling/importance_sampling_ratio/min": 0.7745014488697052,
|
|
"sampling/sampling_logp_difference/max": 0.2783425450325012,
|
|
"sampling/sampling_logp_difference/mean": 0.020684493146836756,
|
|
"step": 10400
|
|
},
|
|
{
|
|
"epoch": 0.9561459961386412,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 299.35602094240835,
|
|
"eval_completions/max_terminated_length": 297.6038394415358,
|
|
"eval_completions/mean_length": 211.33987783595114,
|
|
"eval_completions/mean_terminated_length": 210.99887819939258,
|
|
"eval_completions/min_length": 136.3089005235602,
|
|
"eval_completions/min_terminated_length": 136.3089005235602,
|
|
"eval_entropy": 1.1057024242886282,
|
|
"eval_frac_reward_zero_std": 0.35602094240837695,
|
|
"eval_kl": 0.04526400217302078,
|
|
"eval_loss": 0.002282296773046255,
|
|
"eval_num_tokens": 45250036.0,
|
|
"eval_reward": 0.06697207678883071,
|
|
"eval_reward_std": 0.311346808939288,
|
|
"eval_rewards/reward_fn/mean": 0.06697207678883071,
|
|
"eval_rewards/reward_fn/std": 0.31134681632485484,
|
|
"eval_runtime": 1746.1043,
|
|
"eval_samples_per_second": 0.328,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2714436260193431,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.9999922506680248,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.783341272651213,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2741212345802763,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.021184583444856953,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 10400
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.2,
|
|
"completions/max_terminated_length": 285.2,
|
|
"completions/mean_length": 200.425,
|
|
"completions/mean_terminated_length": 200.425,
|
|
"completions/min_length": 132.1,
|
|
"completions/min_terminated_length": 132.1,
|
|
"entropy": 1.0518465787172318,
|
|
"epoch": 0.9570653672887744,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1126018688082695,
|
|
"kl": 0.04162971572950482,
|
|
"learning_rate": 4.0744919242833245e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 45289734.0,
|
|
"reward": 0.49375,
|
|
"reward_std": 0.42412149012088773,
|
|
"rewards/reward_fn/mean": 0.49375,
|
|
"rewards/reward_fn/std": 0.4241215199232101,
|
|
"sampling/importance_sampling_ratio/max": 1.2854412913322448,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004142999649048,
|
|
"sampling/importance_sampling_ratio/min": 0.7297350227832794,
|
|
"sampling/sampling_logp_difference/max": 0.33484777212142947,
|
|
"sampling/sampling_logp_difference/mean": 0.022272344678640366,
|
|
"step": 10410
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 341.9,
|
|
"completions/max_terminated_length": 341.9,
|
|
"completions/mean_length": 235.8625,
|
|
"completions/mean_terminated_length": 235.8625,
|
|
"completions/min_length": 153.6,
|
|
"completions/min_terminated_length": 153.6,
|
|
"entropy": 1.0714528247714044,
|
|
"epoch": 0.9579847384389077,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 4.253077507019043,
|
|
"kl": 0.03938657914986834,
|
|
"learning_rate": 4.072523098239934e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 45338683.0,
|
|
"reward": -0.0125,
|
|
"reward_std": 0.10606601536273956,
|
|
"rewards/reward_fn/mean": -0.0125,
|
|
"rewards/reward_fn/std": 0.10606601536273956,
|
|
"sampling/importance_sampling_ratio/max": 1.2506747603416444,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004184603691102,
|
|
"sampling/importance_sampling_ratio/min": 0.7712302386760712,
|
|
"sampling/sampling_logp_difference/max": 0.27767720222473147,
|
|
"sampling/sampling_logp_difference/mean": 0.020465991646051406,
|
|
"step": 10420
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 257.1,
|
|
"completions/max_terminated_length": 257.1,
|
|
"completions/mean_length": 186.15,
|
|
"completions/mean_terminated_length": 186.15,
|
|
"completions/min_length": 127.2,
|
|
"completions/min_terminated_length": 127.2,
|
|
"entropy": 1.0581976093351841,
|
|
"epoch": 0.958904109589041,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.12409593164920807,
|
|
"kl": 0.045708527555689216,
|
|
"learning_rate": 4.070552657089268e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 45381039.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.18655259013175965,
|
|
"rewards/reward_fn/mean": 0.175,
|
|
"rewards/reward_fn/std": 0.18655259609222413,
|
|
"sampling/importance_sampling_ratio/max": 1.2400532841682435,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003699898719787,
|
|
"sampling/importance_sampling_ratio/min": 0.8047185063362121,
|
|
"sampling/sampling_logp_difference/max": 0.23777810335159302,
|
|
"sampling/sampling_logp_difference/mean": 0.020782478712499143,
|
|
"step": 10430
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 290.4,
|
|
"completions/max_terminated_length": 290.4,
|
|
"completions/mean_length": 213.3875,
|
|
"completions/mean_terminated_length": 213.3875,
|
|
"completions/min_length": 147.1,
|
|
"completions/min_terminated_length": 147.1,
|
|
"entropy": 1.0435740955173969,
|
|
"epoch": 0.9598234807391745,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.14533483982086182,
|
|
"kl": 0.04180521623347886,
|
|
"learning_rate": 4.068580602855128e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 45420550.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.24434949159622193,
|
|
"rewards/reward_fn/mean": 0.2375,
|
|
"rewards/reward_fn/std": 0.2443494975566864,
|
|
"sampling/importance_sampling_ratio/max": 1.3305577397346497,
|
|
"sampling/importance_sampling_ratio/mean": 0.999878317117691,
|
|
"sampling/importance_sampling_ratio/min": 0.7734332740306854,
|
|
"sampling/sampling_logp_difference/max": 0.3185199499130249,
|
|
"sampling/sampling_logp_difference/mean": 0.020883983559906484,
|
|
"step": 10440
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.1,
|
|
"completions/max_terminated_length": 280.1,
|
|
"completions/mean_length": 201.8125,
|
|
"completions/mean_terminated_length": 201.8125,
|
|
"completions/min_length": 124.1,
|
|
"completions/min_terminated_length": 124.1,
|
|
"entropy": 1.0251258060336113,
|
|
"epoch": 0.9607428518893077,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.106839656829834,
|
|
"kl": 0.05958384588593617,
|
|
"learning_rate": 4.066606937562972e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 45461055.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.32838348150253294,
|
|
"rewards/reward_fn/mean": 0.1375,
|
|
"rewards/reward_fn/std": 0.3283834934234619,
|
|
"sampling/importance_sampling_ratio/max": 1.235090935230255,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003528654575349,
|
|
"sampling/importance_sampling_ratio/min": 0.7531449735164643,
|
|
"sampling/sampling_logp_difference/max": 0.286450457572937,
|
|
"sampling/sampling_logp_difference/mean": 0.02066156528890133,
|
|
"step": 10450
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.6,
|
|
"completions/max_terminated_length": 308.6,
|
|
"completions/mean_length": 206.6125,
|
|
"completions/mean_terminated_length": 206.6125,
|
|
"completions/min_length": 129.2,
|
|
"completions/min_terminated_length": 129.2,
|
|
"entropy": 1.064510666579008,
|
|
"epoch": 0.961662223039441,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.22392216324806213,
|
|
"kl": 0.04095704212086275,
|
|
"learning_rate": 4.064631663239913e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 45501960.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.28112500309944155,
|
|
"rewards/reward_fn/mean": 0.1,
|
|
"rewards/reward_fn/std": 0.28112501502037046,
|
|
"sampling/importance_sampling_ratio/max": 1.3254770159721374,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999226272106171,
|
|
"sampling/importance_sampling_ratio/min": 0.7863228738307952,
|
|
"sampling/sampling_logp_difference/max": 0.2991049289703369,
|
|
"sampling/sampling_logp_difference/mean": 0.02175606582313776,
|
|
"step": 10460
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 328.4,
|
|
"completions/max_terminated_length": 315.4,
|
|
"completions/mean_length": 223.7875,
|
|
"completions/mean_terminated_length": 220.75,
|
|
"completions/min_length": 142.1,
|
|
"completions/min_terminated_length": 142.1,
|
|
"entropy": 1.109752208739519,
|
|
"epoch": 0.9625815941895743,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.12965983152389526,
|
|
"kl": 0.04293912897119299,
|
|
"learning_rate": 4.0626547819147145e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 45550591.0,
|
|
"reward": 0.13125,
|
|
"reward_std": 0.3534108102321625,
|
|
"rewards/reward_fn/mean": 0.13125,
|
|
"rewards/reward_fn/std": 0.3534108340740204,
|
|
"sampling/importance_sampling_ratio/max": 1.2344344854354858,
|
|
"sampling/importance_sampling_ratio/mean": 1.000164932012558,
|
|
"sampling/importance_sampling_ratio/min": 0.755420196056366,
|
|
"sampling/sampling_logp_difference/max": 0.2939692497253418,
|
|
"sampling/sampling_logp_difference/mean": 0.02146379016339779,
|
|
"step": 10470
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 332.6,
|
|
"completions/max_terminated_length": 332.6,
|
|
"completions/mean_length": 234.6875,
|
|
"completions/mean_terminated_length": 234.6875,
|
|
"completions/min_length": 142.3,
|
|
"completions/min_terminated_length": 142.3,
|
|
"entropy": 1.1467086419463157,
|
|
"epoch": 0.9635009653397076,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.1450110524892807,
|
|
"kl": 0.04648222279502079,
|
|
"learning_rate": 4.060676295617793e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 45597862.0,
|
|
"reward": 0.04375,
|
|
"reward_std": 0.380746054649353,
|
|
"rewards/reward_fn/mean": 0.04375,
|
|
"rewards/reward_fn/std": 0.3807460844516754,
|
|
"sampling/importance_sampling_ratio/max": 1.2943558812141418,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997293531894684,
|
|
"sampling/importance_sampling_ratio/min": 0.7739385962486267,
|
|
"sampling/sampling_logp_difference/max": 0.28885374069213865,
|
|
"sampling/sampling_logp_difference/mean": 0.02238720078021288,
|
|
"step": 10480
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 292.5,
|
|
"completions/max_terminated_length": 276.3,
|
|
"completions/mean_length": 200.425,
|
|
"completions/mean_terminated_length": 197.0642868041992,
|
|
"completions/min_length": 119.4,
|
|
"completions/min_terminated_length": 119.4,
|
|
"entropy": 1.0634149111807347,
|
|
"epoch": 0.9644203364898409,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.664721488952637,
|
|
"kl": 0.044651976285967974,
|
|
"learning_rate": 4.058696206381212e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 45640568.0,
|
|
"reward": -0.00625,
|
|
"reward_std": 0.3202360600233078,
|
|
"rewards/reward_fn/mean": -0.00625,
|
|
"rewards/reward_fn/std": 0.3202360659837723,
|
|
"sampling/importance_sampling_ratio/max": 1.307364273071289,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001919090747833,
|
|
"sampling/importance_sampling_ratio/min": 0.7898947954177856,
|
|
"sampling/sampling_logp_difference/max": 0.30578471422195436,
|
|
"sampling/sampling_logp_difference/mean": 0.02079794593155384,
|
|
"step": 10490
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.0,
|
|
"completions/max_terminated_length": 326.0,
|
|
"completions/mean_length": 212.4875,
|
|
"completions/mean_terminated_length": 212.4875,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 1.1407897278666497,
|
|
"epoch": 0.9653397076399742,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.448066711425781,
|
|
"kl": 0.03198553828406148,
|
|
"learning_rate": 4.056714516238683e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 45686087.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.36426674723625185,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.36426675915718076,
|
|
"sampling/importance_sampling_ratio/max": 1.2527343869209289,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003202319145204,
|
|
"sampling/importance_sampling_ratio/min": 0.7954131484031677,
|
|
"sampling/sampling_logp_difference/max": 0.24433987140655516,
|
|
"sampling/sampling_logp_difference/mean": 0.022898581624031068,
|
|
"step": 10500
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.3,
|
|
"completions/max_terminated_length": 316.3,
|
|
"completions/mean_length": 225.7875,
|
|
"completions/mean_terminated_length": 225.7875,
|
|
"completions/min_length": 148.9,
|
|
"completions/min_terminated_length": 148.9,
|
|
"entropy": 1.0881177850067616,
|
|
"epoch": 0.9662590787901075,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 4.399587631225586,
|
|
"kl": 0.048546767968218774,
|
|
"learning_rate": 4.054731227225559e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 45729318.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.2112731784582138,
|
|
"rewards/reward_fn/mean": 0.2,
|
|
"rewards/reward_fn/std": 0.2112731844186783,
|
|
"sampling/importance_sampling_ratio/max": 1.3067634344100951,
|
|
"sampling/importance_sampling_ratio/mean": 1.000279676914215,
|
|
"sampling/importance_sampling_ratio/min": 0.8044389307498931,
|
|
"sampling/sampling_logp_difference/max": 0.271681535243988,
|
|
"sampling/sampling_logp_difference/mean": 0.021586478874087333,
|
|
"step": 10510
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 329.8,
|
|
"completions/max_terminated_length": 329.8,
|
|
"completions/mean_length": 219.2,
|
|
"completions/mean_terminated_length": 219.2,
|
|
"completions/min_length": 125.2,
|
|
"completions/min_terminated_length": 125.2,
|
|
"entropy": 1.143178606033325,
|
|
"epoch": 0.9671784499402408,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1652517020702362,
|
|
"kl": 0.041311096376739444,
|
|
"learning_rate": 4.0527463413788385e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 45773622.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.3155669867992401,
|
|
"rewards/reward_fn/mean": 0.1625,
|
|
"rewards/reward_fn/std": 0.31556699275970457,
|
|
"sampling/importance_sampling_ratio/max": 1.2878233075141907,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004111528396606,
|
|
"sampling/importance_sampling_ratio/min": 0.7516210079193115,
|
|
"sampling/sampling_logp_difference/max": 0.3114285707473755,
|
|
"sampling/sampling_logp_difference/mean": 0.021550540812313557,
|
|
"step": 10520
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.7,
|
|
"completions/max_terminated_length": 302.7,
|
|
"completions/mean_length": 216.7875,
|
|
"completions/mean_terminated_length": 216.7875,
|
|
"completions/min_length": 139.5,
|
|
"completions/min_terminated_length": 139.5,
|
|
"entropy": 1.134613348543644,
|
|
"epoch": 0.9680978210903742,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.383565902709961,
|
|
"kl": 0.04815573546802625,
|
|
"learning_rate": 4.050759860737158e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 45817549.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.2828427076339722,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.2828427076339722,
|
|
"sampling/importance_sampling_ratio/max": 1.2433046221733093,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996812045574188,
|
|
"sampling/importance_sampling_ratio/min": 0.7778947591781616,
|
|
"sampling/sampling_logp_difference/max": 0.26329495906829836,
|
|
"sampling/sampling_logp_difference/mean": 0.02150979358702898,
|
|
"step": 10530
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.6,
|
|
"completions/max_terminated_length": 277.6,
|
|
"completions/mean_length": 198.5875,
|
|
"completions/mean_terminated_length": 198.5875,
|
|
"completions/min_length": 131.4,
|
|
"completions/min_terminated_length": 131.4,
|
|
"entropy": 1.0837464720010757,
|
|
"epoch": 0.9690171922405075,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.17261937260627747,
|
|
"kl": 0.03875091255758889,
|
|
"learning_rate": 4.0487717873407905e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 45861508.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/reward_fn/mean": 0.15,
|
|
"rewards/reward_fn/std": 0.27396377325057986,
|
|
"sampling/importance_sampling_ratio/max": 1.2494304895401,
|
|
"sampling/importance_sampling_ratio/mean": 1.000165432691574,
|
|
"sampling/importance_sampling_ratio/min": 0.8084233701229095,
|
|
"sampling/sampling_logp_difference/max": 0.23188527822494506,
|
|
"sampling/sampling_logp_difference/mean": 0.020969351194798945,
|
|
"step": 10540
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 328.2,
|
|
"completions/max_terminated_length": 328.2,
|
|
"completions/mean_length": 225.925,
|
|
"completions/mean_terminated_length": 225.925,
|
|
"completions/min_length": 132.3,
|
|
"completions/min_terminated_length": 132.3,
|
|
"entropy": 1.1044180184602737,
|
|
"epoch": 0.9699365633906408,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.855181694030762,
|
|
"kl": 0.03996765591436997,
|
|
"learning_rate": 4.046782123231649e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 45904718.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.332049286365509,
|
|
"rewards/reward_fn/mean": 0.0875,
|
|
"rewards/reward_fn/std": 0.3320492923259735,
|
|
"sampling/importance_sampling_ratio/max": 1.2515955567359924,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004275441169739,
|
|
"sampling/importance_sampling_ratio/min": 0.7667010009288788,
|
|
"sampling/sampling_logp_difference/max": 0.2769248247146606,
|
|
"sampling/sampling_logp_difference/mean": 0.02228622790426016,
|
|
"step": 10550
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 293.5,
|
|
"completions/max_terminated_length": 293.5,
|
|
"completions/mean_length": 209.025,
|
|
"completions/mean_terminated_length": 209.025,
|
|
"completions/min_length": 145.1,
|
|
"completions/min_terminated_length": 145.1,
|
|
"entropy": 1.0964238591492177,
|
|
"epoch": 0.9708559345407741,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0910649523139,
|
|
"kl": 0.04920785235590301,
|
|
"learning_rate": 4.044790870453277e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 45948888.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.2927524596452713,
|
|
"rewards/reward_fn/mean": 0.25,
|
|
"rewards/reward_fn/std": 0.29275247156620027,
|
|
"sampling/importance_sampling_ratio/max": 1.229606318473816,
|
|
"sampling/importance_sampling_ratio/mean": 1.0000321090221405,
|
|
"sampling/importance_sampling_ratio/min": 0.7531989216804504,
|
|
"sampling/sampling_logp_difference/max": 0.2971944689750671,
|
|
"sampling/sampling_logp_difference/mean": 0.02184291649609804,
|
|
"step": 10560
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.6,
|
|
"completions/max_terminated_length": 286.6,
|
|
"completions/mean_length": 192.7,
|
|
"completions/mean_terminated_length": 192.7,
|
|
"completions/min_length": 125.0,
|
|
"completions/min_terminated_length": 125.0,
|
|
"entropy": 1.0779813207685947,
|
|
"epoch": 0.9717753056909074,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.180692672729492,
|
|
"kl": 0.03135224454454146,
|
|
"learning_rate": 4.042798031050851e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 45990120.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.2857582807540894,
|
|
"rewards/reward_fn/mean": 0.3625,
|
|
"rewards/reward_fn/std": 0.28575828671455383,
|
|
"sampling/importance_sampling_ratio/max": 1.285517978668213,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002331852912902,
|
|
"sampling/importance_sampling_ratio/min": 0.7724617063999176,
|
|
"sampling/sampling_logp_difference/max": 0.31488857269287107,
|
|
"sampling/sampling_logp_difference/mean": 0.021045699529349805,
|
|
"step": 10570
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.6,
|
|
"completions/max_terminated_length": 315.6,
|
|
"completions/mean_length": 222.5125,
|
|
"completions/mean_terminated_length": 222.5125,
|
|
"completions/min_length": 145.6,
|
|
"completions/min_terminated_length": 145.6,
|
|
"entropy": 1.1628859020769595,
|
|
"epoch": 0.9726946768410407,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 6.529232978820801,
|
|
"kl": 0.05697717809816823,
|
|
"learning_rate": 4.040803607071177e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 46036321.0,
|
|
"reward": 0.04375,
|
|
"reward_std": 0.27085447013378144,
|
|
"rewards/reward_fn/mean": 0.04375,
|
|
"rewards/reward_fn/std": 0.2708544820547104,
|
|
"sampling/importance_sampling_ratio/max": 1.2764482140541076,
|
|
"sampling/importance_sampling_ratio/mean": 0.9996897161006928,
|
|
"sampling/importance_sampling_ratio/min": 0.7757793724536896,
|
|
"sampling/sampling_logp_difference/max": 0.28104729056358335,
|
|
"sampling/sampling_logp_difference/mean": 0.02207276560366154,
|
|
"step": 10580
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.6,
|
|
"completions/max_terminated_length": 325.6,
|
|
"completions/mean_length": 212.125,
|
|
"completions/mean_terminated_length": 212.125,
|
|
"completions/min_length": 132.8,
|
|
"completions/min_terminated_length": 132.8,
|
|
"entropy": 1.060155439376831,
|
|
"epoch": 0.973614047991174,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.14923733472824097,
|
|
"kl": 0.03344500265084207,
|
|
"learning_rate": 4.038807600562686e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 46083755.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.26074906885623933,
|
|
"rewards/reward_fn/mean": 0.2875,
|
|
"rewards/reward_fn/std": 0.2607490807771683,
|
|
"sampling/importance_sampling_ratio/max": 1.2602178454399109,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003794193267823,
|
|
"sampling/importance_sampling_ratio/min": 0.7772373974323272,
|
|
"sampling/sampling_logp_difference/max": 0.2759662985801697,
|
|
"sampling/sampling_logp_difference/mean": 0.021500770561397077,
|
|
"step": 10590
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.0,
|
|
"completions/max_terminated_length": 306.0,
|
|
"completions/mean_length": 214.35,
|
|
"completions/mean_terminated_length": 214.35,
|
|
"completions/min_length": 138.0,
|
|
"completions/min_terminated_length": 138.0,
|
|
"entropy": 1.1174485310912132,
|
|
"epoch": 0.9745334191413073,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.09379025548696518,
|
|
"kl": 0.05338641300913878,
|
|
"learning_rate": 4.036810013575441e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 46126295.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.4027158319950104,
|
|
"rewards/reward_fn/mean": 0.0625,
|
|
"rewards/reward_fn/std": 0.4027158319950104,
|
|
"sampling/importance_sampling_ratio/max": 1.2558752417564392,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998934984207153,
|
|
"sampling/importance_sampling_ratio/min": 0.7756098926067352,
|
|
"sampling/sampling_logp_difference/max": 0.27413382530212405,
|
|
"sampling/sampling_logp_difference/mean": 0.021913588047027588,
|
|
"step": 10600
|
|
},
|
|
{
|
|
"epoch": 0.9745334191413073,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.002617801047120419,
|
|
"eval_completions/max_length": 305.5776614310646,
|
|
"eval_completions/max_terminated_length": 302.38568935427577,
|
|
"eval_completions/mean_length": 210.04363001745202,
|
|
"eval_completions/mean_terminated_length": 209.3403247879854,
|
|
"eval_completions/min_length": 133.1937172774869,
|
|
"eval_completions/min_terminated_length": 133.1937172774869,
|
|
"eval_entropy": 1.1039680640422862,
|
|
"eval_frac_reward_zero_std": 0.36300174520069806,
|
|
"eval_kl": 0.0462461728773309,
|
|
"eval_loss": 0.0024066148325800896,
|
|
"eval_num_tokens": 46126295.0,
|
|
"eval_reward": 0.0650087260034904,
|
|
"eval_reward_std": 0.31234918594568395,
|
|
"eval_rewards/reward_fn/mean": 0.0650087260034904,
|
|
"eval_rewards/reward_fn/std": 0.31234919437147146,
|
|
"eval_runtime": 1776.5787,
|
|
"eval_samples_per_second": 0.323,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2697018655390848,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000249891172945,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7778603796039368,
|
|
"eval_sampling/sampling_logp_difference/max": 0.27872646494685666,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02105851740256134,
|
|
"eval_steps_per_second": 0.041,
|
|
"step": 10600
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 272.0,
|
|
"completions/max_terminated_length": 272.0,
|
|
"completions/mean_length": 183.425,
|
|
"completions/mean_terminated_length": 183.425,
|
|
"completions/min_length": 110.0,
|
|
"completions/min_terminated_length": 110.0,
|
|
"entropy": 1.0763877734541893,
|
|
"epoch": 0.9754527902914406,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 8.570212364196777,
|
|
"kl": 0.05944876414141618,
|
|
"learning_rate": 4.034810848161121e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 46167705.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.2945584118366241,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.2945584237575531,
|
|
"sampling/importance_sampling_ratio/max": 1.2308587670326232,
|
|
"sampling/importance_sampling_ratio/mean": 1.0003653705120086,
|
|
"sampling/importance_sampling_ratio/min": 0.7976280927658081,
|
|
"sampling/sampling_logp_difference/max": 0.25031859874725343,
|
|
"sampling/sampling_logp_difference/mean": 0.020743202045559885,
|
|
"step": 10610
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.8,
|
|
"completions/max_terminated_length": 276.8,
|
|
"completions/mean_length": 201.225,
|
|
"completions/mean_terminated_length": 201.225,
|
|
"completions/min_length": 132.4,
|
|
"completions/min_terminated_length": 132.4,
|
|
"entropy": 1.0714590944349767,
|
|
"epoch": 0.976372161441574,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.08955306559801102,
|
|
"kl": 0.03883964841370471,
|
|
"learning_rate": 4.03281010637303e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 46210043.0,
|
|
"reward": -0.4125,
|
|
"reward_std": 0.30646984577178954,
|
|
"rewards/reward_fn/mean": -0.4125,
|
|
"rewards/reward_fn/std": 0.3064698576927185,
|
|
"sampling/importance_sampling_ratio/max": 1.2699307084083558,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997348785400391,
|
|
"sampling/importance_sampling_ratio/min": 0.793196564912796,
|
|
"sampling/sampling_logp_difference/max": 0.26364885568618773,
|
|
"sampling/sampling_logp_difference/mean": 0.021123329736292364,
|
|
"step": 10620
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 308.2,
|
|
"completions/max_terminated_length": 286.6,
|
|
"completions/mean_length": 210.3,
|
|
"completions/mean_terminated_length": 206.93392944335938,
|
|
"completions/min_length": 145.9,
|
|
"completions/min_terminated_length": 145.9,
|
|
"entropy": 1.1149389427155256,
|
|
"epoch": 0.9772915325917073,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.12780416011810303,
|
|
"kl": 0.05383593825972639,
|
|
"learning_rate": 4.030807790266089e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 46254755.0,
|
|
"reward": 0.14375,
|
|
"reward_std": 0.4107950896024704,
|
|
"rewards/reward_fn/mean": 0.14375,
|
|
"rewards/reward_fn/std": 0.41079509556293486,
|
|
"sampling/importance_sampling_ratio/max": 1.2807302713394164,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001554787158966,
|
|
"sampling/importance_sampling_ratio/min": 0.7978473007678986,
|
|
"sampling/sampling_logp_difference/max": 0.2565577030181885,
|
|
"sampling/sampling_logp_difference/mean": 0.021979923360049725,
|
|
"step": 10630
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 308.4,
|
|
"completions/max_terminated_length": 291.7,
|
|
"completions/mean_length": 224.325,
|
|
"completions/mean_terminated_length": 221.30892944335938,
|
|
"completions/min_length": 151.9,
|
|
"completions/min_terminated_length": 151.9,
|
|
"entropy": 1.1214619420468808,
|
|
"epoch": 0.9782109037418406,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 5.0095648765563965,
|
|
"kl": 0.045149215031415224,
|
|
"learning_rate": 4.028803901896839e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 46299261.0,
|
|
"reward": 0.10625,
|
|
"reward_std": 0.3626292645931244,
|
|
"rewards/reward_fn/mean": 0.10625,
|
|
"rewards/reward_fn/std": 0.3626292645931244,
|
|
"sampling/importance_sampling_ratio/max": 1.230244553089142,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998919427394867,
|
|
"sampling/importance_sampling_ratio/min": 0.7904980719089508,
|
|
"sampling/sampling_logp_difference/max": 0.23734110593795776,
|
|
"sampling/sampling_logp_difference/mean": 0.021747793443500994,
|
|
"step": 10640
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 253.1,
|
|
"completions/max_terminated_length": 253.1,
|
|
"completions/mean_length": 183.7875,
|
|
"completions/mean_terminated_length": 183.7875,
|
|
"completions/min_length": 116.1,
|
|
"completions/min_terminated_length": 116.1,
|
|
"entropy": 0.9508635349571705,
|
|
"epoch": 0.9791302748919739,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.13384227454662323,
|
|
"kl": 0.037569528320454994,
|
|
"learning_rate": 4.026798443323433e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 46337140.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.10606601536273956,
|
|
"rewards/reward_fn/mean": 0.4625,
|
|
"rewards/reward_fn/std": 0.10606601536273956,
|
|
"sampling/importance_sampling_ratio/max": 1.2932259559631347,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002963900566102,
|
|
"sampling/importance_sampling_ratio/min": 0.7995527446269989,
|
|
"sampling/sampling_logp_difference/max": 0.26656681299209595,
|
|
"sampling/sampling_logp_difference/mean": 0.019971209997311234,
|
|
"step": 10650
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 348.7,
|
|
"completions/max_terminated_length": 321.9,
|
|
"completions/mean_length": 230.3,
|
|
"completions/mean_terminated_length": 224.51071472167968,
|
|
"completions/min_length": 139.1,
|
|
"completions/min_terminated_length": 139.1,
|
|
"entropy": 1.124989303201437,
|
|
"epoch": 0.9800496460421072,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.871088027954102,
|
|
"kl": 0.04560805626679212,
|
|
"learning_rate": 4.024791416605638e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 46381876.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.3611987292766571,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.3611987292766571,
|
|
"sampling/importance_sampling_ratio/max": 1.2567938923835755,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997237861156464,
|
|
"sampling/importance_sampling_ratio/min": 0.7835067808628082,
|
|
"sampling/sampling_logp_difference/max": 0.2627923250198364,
|
|
"sampling/sampling_logp_difference/mean": 0.021575084142386913,
|
|
"step": 10660
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.1,
|
|
"completions/max_terminated_length": 314.1,
|
|
"completions/mean_length": 231.225,
|
|
"completions/mean_terminated_length": 231.225,
|
|
"completions/min_length": 146.9,
|
|
"completions/min_terminated_length": 146.9,
|
|
"entropy": 1.0659235760569572,
|
|
"epoch": 0.9809690171922405,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 4.913619041442871,
|
|
"kl": 0.04187723956420086,
|
|
"learning_rate": 4.022782823804832e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 46425430.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.3446744382381439,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.3446744501590729,
|
|
"sampling/importance_sampling_ratio/max": 1.290179181098938,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999665856361389,
|
|
"sampling/importance_sampling_ratio/min": 0.769069892168045,
|
|
"sampling/sampling_logp_difference/max": 0.2794591188430786,
|
|
"sampling/sampling_logp_difference/mean": 0.021413156390190126,
|
|
"step": 10670
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 260.6,
|
|
"completions/max_terminated_length": 260.6,
|
|
"completions/mean_length": 192.0875,
|
|
"completions/mean_terminated_length": 192.0875,
|
|
"completions/min_length": 132.6,
|
|
"completions/min_terminated_length": 132.6,
|
|
"entropy": 1.0125107638537885,
|
|
"epoch": 0.9818883883423738,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.907232761383057,
|
|
"kl": 0.03358312457276043,
|
|
"learning_rate": 4.0207726669839984e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 46472781.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.3282748609781265,
|
|
"rewards/reward_fn/mean": 0.075,
|
|
"rewards/reward_fn/std": 0.328274866938591,
|
|
"sampling/importance_sampling_ratio/max": 1.2436286211013794,
|
|
"sampling/importance_sampling_ratio/mean": 1.000118726491928,
|
|
"sampling/importance_sampling_ratio/min": 0.7986629605293274,
|
|
"sampling/sampling_logp_difference/max": 0.25396203994750977,
|
|
"sampling/sampling_logp_difference/mean": 0.02032058909535408,
|
|
"step": 10680
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.4,
|
|
"completions/max_terminated_length": 319.4,
|
|
"completions/mean_length": 224.35,
|
|
"completions/mean_terminated_length": 224.35,
|
|
"completions/min_length": 152.6,
|
|
"completions/min_terminated_length": 152.6,
|
|
"entropy": 1.0958833031356334,
|
|
"epoch": 0.9828077594925071,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.126735687255859,
|
|
"kl": 0.048667197313625364,
|
|
"learning_rate": 4.018760948207731e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 46516737.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.3093269467353821,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.3093269467353821,
|
|
"sampling/importance_sampling_ratio/max": 1.3144049048423767,
|
|
"sampling/importance_sampling_ratio/mean": 1.0004535794258118,
|
|
"sampling/importance_sampling_ratio/min": 0.7688483357429504,
|
|
"sampling/sampling_logp_difference/max": 0.3154332756996155,
|
|
"sampling/sampling_logp_difference/mean": 0.021407747454941273,
|
|
"step": 10690
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 334.8,
|
|
"completions/max_terminated_length": 334.8,
|
|
"completions/mean_length": 227.925,
|
|
"completions/mean_terminated_length": 227.925,
|
|
"completions/min_length": 152.1,
|
|
"completions/min_terminated_length": 152.1,
|
|
"entropy": 1.0676656879484654,
|
|
"epoch": 0.9837271306426404,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.12427646666765213,
|
|
"kl": 0.04723160498542711,
|
|
"learning_rate": 4.016747669542227e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 46562275.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.38434176445007323,
|
|
"rewards/reward_fn/mean": 0.3625,
|
|
"rewards/reward_fn/std": 0.3843417763710022,
|
|
"sampling/importance_sampling_ratio/max": 1.2663423061370849,
|
|
"sampling/importance_sampling_ratio/mean": 1.000032937526703,
|
|
"sampling/importance_sampling_ratio/min": 0.7741970658302307,
|
|
"sampling/sampling_logp_difference/max": 0.28989644050598146,
|
|
"sampling/sampling_logp_difference/mean": 0.020921942219138144,
|
|
"step": 10700
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 329.0,
|
|
"completions/max_terminated_length": 318.2,
|
|
"completions/mean_length": 229.5875,
|
|
"completions/mean_terminated_length": 226.76607360839844,
|
|
"completions/min_length": 144.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"entropy": 1.0542023926973343,
|
|
"epoch": 0.9846465017927737,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 6.42956018447876,
|
|
"kl": 0.050429430173244326,
|
|
"learning_rate": 4.014732833055284e-06,
|
|
"loss": 0.0028,
|
|
"num_tokens": 46606546.0,
|
|
"reward": -0.14375,
|
|
"reward_std": 0.3627240300178528,
|
|
"rewards/reward_fn/mean": -0.14375,
|
|
"rewards/reward_fn/std": 0.3627240478992462,
|
|
"sampling/importance_sampling_ratio/max": 1.2930376052856445,
|
|
"sampling/importance_sampling_ratio/mean": 0.9997085988521576,
|
|
"sampling/importance_sampling_ratio/min": 0.770572566986084,
|
|
"sampling/sampling_logp_difference/max": 0.29186494946479796,
|
|
"sampling/sampling_logp_difference/mean": 0.020957361534237863,
|
|
"step": 10710
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 252.4,
|
|
"completions/max_terminated_length": 252.4,
|
|
"completions/mean_length": 187.5375,
|
|
"completions/mean_terminated_length": 187.5375,
|
|
"completions/min_length": 124.0,
|
|
"completions/min_terminated_length": 124.0,
|
|
"entropy": 1.0119339033961297,
|
|
"epoch": 0.9855658729429071,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.12501323223114014,
|
|
"kl": 0.03492223062203266,
|
|
"learning_rate": 4.0127164408163e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 46645541.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.2230676978826523,
|
|
"rewards/reward_fn/mean": 0.2125,
|
|
"rewards/reward_fn/std": 0.2230676978826523,
|
|
"sampling/importance_sampling_ratio/max": 1.3230002403259278,
|
|
"sampling/importance_sampling_ratio/mean": 1.0002218723297118,
|
|
"sampling/importance_sampling_ratio/min": 0.756602555513382,
|
|
"sampling/sampling_logp_difference/max": 0.328651225566864,
|
|
"sampling/sampling_logp_difference/mean": 0.02056221794337034,
|
|
"step": 10720
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.5,
|
|
"completions/max_terminated_length": 315.5,
|
|
"completions/mean_length": 217.125,
|
|
"completions/mean_terminated_length": 217.125,
|
|
"completions/min_length": 146.7,
|
|
"completions/min_terminated_length": 146.7,
|
|
"entropy": 1.0627942934632302,
|
|
"epoch": 0.9864852440930404,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 7.318994522094727,
|
|
"kl": 0.06559816369554028,
|
|
"learning_rate": 4.010698494896274e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 46686871.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.28489942848682404,
|
|
"rewards/reward_fn/mean": 0.0125,
|
|
"rewards/reward_fn/std": 0.284899440407753,
|
|
"sampling/importance_sampling_ratio/max": 1.245354425907135,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001755833625794,
|
|
"sampling/importance_sampling_ratio/min": 0.7547447919845581,
|
|
"sampling/sampling_logp_difference/max": 0.290567022562027,
|
|
"sampling/sampling_logp_difference/mean": 0.021006381325423717,
|
|
"step": 10730
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 322.9,
|
|
"completions/max_terminated_length": 322.9,
|
|
"completions/mean_length": 226.175,
|
|
"completions/mean_terminated_length": 226.175,
|
|
"completions/min_length": 143.2,
|
|
"completions/min_terminated_length": 143.2,
|
|
"entropy": 1.1587119542062283,
|
|
"epoch": 0.9874046152431737,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 4.134674549102783,
|
|
"kl": 0.03835391944739967,
|
|
"learning_rate": 4.008678997367795e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 46733165.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.31933356523513795,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.31933358311653137,
|
|
"sampling/importance_sampling_ratio/max": 1.2341816663742065,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999795913696289,
|
|
"sampling/importance_sampling_ratio/min": 0.7795907139778138,
|
|
"sampling/sampling_logp_difference/max": 0.2498707354068756,
|
|
"sampling/sampling_logp_difference/mean": 0.021864561550319194,
|
|
"step": 10740
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 277.7,
|
|
"completions/max_terminated_length": 277.7,
|
|
"completions/mean_length": 194.8875,
|
|
"completions/mean_terminated_length": 194.8875,
|
|
"completions/min_length": 133.3,
|
|
"completions/min_terminated_length": 133.3,
|
|
"entropy": 1.0575469985604287,
|
|
"epoch": 0.988323986393307,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 6.4292192459106445,
|
|
"kl": 0.04080540960421786,
|
|
"learning_rate": 4.006657950305051e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 46775636.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.26819893419742585,
|
|
"rewards/reward_fn/mean": 0.1125,
|
|
"rewards/reward_fn/std": 0.2681989401578903,
|
|
"sampling/importance_sampling_ratio/max": 1.3108070492744446,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998845219612121,
|
|
"sampling/importance_sampling_ratio/min": 0.8049184024333954,
|
|
"sampling/sampling_logp_difference/max": 0.2715925097465515,
|
|
"sampling/sampling_logp_difference/mean": 0.02117980606853962,
|
|
"step": 10750
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 289.4,
|
|
"completions/max_terminated_length": 289.4,
|
|
"completions/mean_length": 206.5,
|
|
"completions/mean_terminated_length": 206.5,
|
|
"completions/min_length": 131.6,
|
|
"completions/min_terminated_length": 131.6,
|
|
"entropy": 1.0935260951519012,
|
|
"epoch": 0.9892433575434403,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 5.3015055656433105,
|
|
"kl": 0.05091552591184154,
|
|
"learning_rate": 4.0046353557838165e-06,
|
|
"loss": 0.0026,
|
|
"num_tokens": 46821516.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.2958350956439972,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.29583510756492615,
|
|
"sampling/importance_sampling_ratio/max": 1.2403937816619872,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999417185783386,
|
|
"sampling/importance_sampling_ratio/min": 0.8000710666179657,
|
|
"sampling/sampling_logp_difference/max": 0.23776893615722655,
|
|
"sampling/sampling_logp_difference/mean": 0.020979249477386476,
|
|
"step": 10760
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 313.6,
|
|
"completions/max_terminated_length": 313.6,
|
|
"completions/mean_length": 220.6625,
|
|
"completions/mean_terminated_length": 220.6625,
|
|
"completions/min_length": 143.5,
|
|
"completions/min_terminated_length": 143.5,
|
|
"entropy": 1.0781245514750482,
|
|
"epoch": 0.9901627286935736,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.216675281524658,
|
|
"kl": 0.04917086389614269,
|
|
"learning_rate": 4.0026112158814624e-06,
|
|
"loss": 0.0029,
|
|
"num_tokens": 46864977.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.35166861712932584,
|
|
"rewards/reward_fn/mean": 0.2125,
|
|
"rewards/reward_fn/std": 0.3516686350107193,
|
|
"sampling/importance_sampling_ratio/max": 1.2704369664192199,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001218438148498,
|
|
"sampling/importance_sampling_ratio/min": 0.8047501444816589,
|
|
"sampling/sampling_logp_difference/max": 0.25602006912231445,
|
|
"sampling/sampling_logp_difference/mean": 0.021384000964462756,
|
|
"step": 10770
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.4,
|
|
"completions/max_terminated_length": 303.4,
|
|
"completions/mean_length": 212.075,
|
|
"completions/mean_terminated_length": 212.075,
|
|
"completions/min_length": 154.8,
|
|
"completions/min_terminated_length": 154.8,
|
|
"entropy": 1.1556730784475804,
|
|
"epoch": 0.9910820998437069,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 6.765624523162842,
|
|
"kl": 0.04916229965747334,
|
|
"learning_rate": 4.00058553267694e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 46912055.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.4377940118312836,
|
|
"rewards/reward_fn/mean": 0.1875,
|
|
"rewards/reward_fn/std": 0.43779402375221255,
|
|
"sampling/importance_sampling_ratio/max": 1.2556164026260377,
|
|
"sampling/importance_sampling_ratio/mean": 1.000164794921875,
|
|
"sampling/importance_sampling_ratio/min": 0.7557412564754487,
|
|
"sampling/sampling_logp_difference/max": 0.3009467363357544,
|
|
"sampling/sampling_logp_difference/mean": 0.022604018449783325,
|
|
"step": 10780
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 274.1,
|
|
"completions/max_terminated_length": 274.1,
|
|
"completions/mean_length": 192.0875,
|
|
"completions/mean_terminated_length": 192.0875,
|
|
"completions/min_length": 122.6,
|
|
"completions/min_terminated_length": 122.6,
|
|
"entropy": 1.0514204286038875,
|
|
"epoch": 0.9920014709938402,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.11940952390432358,
|
|
"kl": 0.045541573059745134,
|
|
"learning_rate": 3.99855830825079e-06,
|
|
"loss": 0.0023,
|
|
"num_tokens": 46954750.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.25756419003009795,
|
|
"sampling/importance_sampling_ratio/max": 1.2530922293663025,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998751640319824,
|
|
"sampling/importance_sampling_ratio/min": 0.790794450044632,
|
|
"sampling/sampling_logp_difference/max": 0.25152078866958616,
|
|
"sampling/sampling_logp_difference/mean": 0.02078450843691826,
|
|
"step": 10790
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.5,
|
|
"completions/max_terminated_length": 304.5,
|
|
"completions/mean_length": 204.9,
|
|
"completions/mean_terminated_length": 204.9,
|
|
"completions/min_length": 137.3,
|
|
"completions/min_terminated_length": 137.3,
|
|
"entropy": 1.0131013922393322,
|
|
"epoch": 0.9929208421439735,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.889503479003906,
|
|
"kl": 0.04277862436138093,
|
|
"learning_rate": 3.996529544685132e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 46994526.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.4935811996459961,
|
|
"rewards/reward_fn/mean": 0.025,
|
|
"rewards/reward_fn/std": 0.493581211566925,
|
|
"sampling/importance_sampling_ratio/max": 1.267315721511841,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999509453773499,
|
|
"sampling/importance_sampling_ratio/min": 0.7814526081085205,
|
|
"sampling/sampling_logp_difference/max": 0.2762034058570862,
|
|
"sampling/sampling_logp_difference/mean": 0.020524901524186134,
|
|
"step": 10800
|
|
},
|
|
{
|
|
"epoch": 0.9929208421439735,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.0013089005235602095,
|
|
"eval_completions/max_length": 293.4694589877836,
|
|
"eval_completions/max_terminated_length": 291.9528795811518,
|
|
"eval_completions/mean_length": 203.68760907504364,
|
|
"eval_completions/mean_terminated_length": 203.33972213239153,
|
|
"eval_completions/min_length": 129.20942408376965,
|
|
"eval_completions/min_terminated_length": 129.20942408376965,
|
|
"eval_entropy": 1.1081976287235973,
|
|
"eval_frac_reward_zero_std": 0.3368237347294939,
|
|
"eval_kl": 0.03997780676007947,
|
|
"eval_loss": 0.002060601022094488,
|
|
"eval_num_tokens": 46994526.0,
|
|
"eval_reward": 0.06479057591623037,
|
|
"eval_reward_std": 0.33846851762052604,
|
|
"eval_rewards/reward_fn/mean": 0.06479057591623037,
|
|
"eval_rewards/reward_fn/std": 0.338468528334799,
|
|
"eval_runtime": 1716.124,
|
|
"eval_samples_per_second": 0.334,
|
|
"eval_sampling/importance_sampling_ratio/max": 1.2655667663660766,
|
|
"eval_sampling/importance_sampling_ratio/mean": 1.0000561010775142,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.7830570527916387,
|
|
"eval_sampling/sampling_logp_difference/max": 0.2704578622733527,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.02117616586861758,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 10800
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.9,
|
|
"completions/max_terminated_length": 283.9,
|
|
"completions/mean_length": 203.9375,
|
|
"completions/mean_terminated_length": 203.9375,
|
|
"completions/min_length": 136.6,
|
|
"completions/min_terminated_length": 136.6,
|
|
"entropy": 0.9913013383746148,
|
|
"epoch": 0.9938402132941069,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 5.941771984100342,
|
|
"kl": 0.03260976006858982,
|
|
"learning_rate": 3.9944992440636705e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 47035569.0,
|
|
"reward": 0.4875,
|
|
"reward_std": 0.2394672751426697,
|
|
"rewards/reward_fn/mean": 0.4875,
|
|
"rewards/reward_fn/std": 0.23946728110313414,
|
|
"sampling/importance_sampling_ratio/max": 1.2419529557228088,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998937308788299,
|
|
"sampling/importance_sampling_ratio/min": 0.7698903560638428,
|
|
"sampling/sampling_logp_difference/max": 0.2705738663673401,
|
|
"sampling/sampling_logp_difference/mean": 0.020100896432995796,
|
|
"step": 10810
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 267.0,
|
|
"completions/max_terminated_length": 267.0,
|
|
"completions/mean_length": 188.2625,
|
|
"completions/mean_terminated_length": 188.2625,
|
|
"completions/min_length": 125.3,
|
|
"completions/min_terminated_length": 125.3,
|
|
"entropy": 1.075051011890173,
|
|
"epoch": 0.9947595844442402,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 7.5150017738342285,
|
|
"kl": 0.033782070776214826,
|
|
"learning_rate": 3.992467408471686e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 47073838.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.22598327100276946,
|
|
"rewards/reward_fn/mean": 0.0,
|
|
"rewards/reward_fn/std": 0.22598327696323395,
|
|
"sampling/importance_sampling_ratio/max": 1.2625146150588988,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999453842639923,
|
|
"sampling/importance_sampling_ratio/min": 0.7921305418014526,
|
|
"sampling/sampling_logp_difference/max": 0.258840012550354,
|
|
"sampling/sampling_logp_difference/mean": 0.0216575151309371,
|
|
"step": 10820
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.6,
|
|
"completions/max_terminated_length": 296.6,
|
|
"completions/mean_length": 197.6,
|
|
"completions/mean_terminated_length": 197.6,
|
|
"completions/min_length": 123.5,
|
|
"completions/min_terminated_length": 123.5,
|
|
"entropy": 1.1157053284347058,
|
|
"epoch": 0.9956789555943735,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 5.532589912414551,
|
|
"kl": 0.0465478798490949,
|
|
"learning_rate": 3.990434039996038e-06,
|
|
"loss": 0.0017,
|
|
"num_tokens": 47115718.0,
|
|
"reward": -0.1,
|
|
"reward_std": 0.4555900037288666,
|
|
"rewards/reward_fn/mean": -0.1,
|
|
"rewards/reward_fn/std": 0.4555900156497955,
|
|
"sampling/importance_sampling_ratio/max": 1.266064465045929,
|
|
"sampling/importance_sampling_ratio/mean": 0.9998319029808045,
|
|
"sampling/importance_sampling_ratio/min": 0.7836755692958832,
|
|
"sampling/sampling_logp_difference/max": 0.27566248178482056,
|
|
"sampling/sampling_logp_difference/mean": 0.021598699688911437,
|
|
"step": 10830
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.5,
|
|
"completions/max_terminated_length": 286.5,
|
|
"completions/mean_length": 203.6625,
|
|
"completions/mean_terminated_length": 203.6625,
|
|
"completions/min_length": 125.9,
|
|
"completions/min_terminated_length": 125.9,
|
|
"entropy": 1.0822812251746654,
|
|
"epoch": 0.9965983267445068,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 6.528209209442139,
|
|
"kl": 0.04015933805494569,
|
|
"learning_rate": 3.988399140725157e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 47156683.0,
|
|
"reward": -0.025,
|
|
"reward_std": 0.5010453939437867,
|
|
"rewards/reward_fn/mean": -0.025,
|
|
"rewards/reward_fn/std": 0.50104541182518,
|
|
"sampling/importance_sampling_ratio/max": 1.3065037608146668,
|
|
"sampling/importance_sampling_ratio/mean": 0.999961006641388,
|
|
"sampling/importance_sampling_ratio/min": 0.7814095377922058,
|
|
"sampling/sampling_logp_difference/max": 0.29769723415374755,
|
|
"sampling/sampling_logp_difference/mean": 0.02156674899160862,
|
|
"step": 10840
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 295.0,
|
|
"completions/max_terminated_length": 295.0,
|
|
"completions/mean_length": 201.8375,
|
|
"completions/mean_terminated_length": 201.8375,
|
|
"completions/min_length": 133.7,
|
|
"completions/min_terminated_length": 133.7,
|
|
"entropy": 1.1300729371607303,
|
|
"epoch": 0.9975176978946401,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 7.41755485534668,
|
|
"kl": 0.03942642385372892,
|
|
"learning_rate": 3.98636271274905e-06,
|
|
"loss": 0.0021,
|
|
"num_tokens": 47202126.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3095962584018707,
|
|
"rewards/reward_fn/mean": 0.325,
|
|
"rewards/reward_fn/std": 0.3095962584018707,
|
|
"sampling/importance_sampling_ratio/max": 1.2735634922981263,
|
|
"sampling/importance_sampling_ratio/mean": 1.000087285041809,
|
|
"sampling/importance_sampling_ratio/min": 0.8021250605583191,
|
|
"sampling/sampling_logp_difference/max": 0.25763792991638185,
|
|
"sampling/sampling_logp_difference/mean": 0.02226375211030245,
|
|
"step": 10850
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 302.2,
|
|
"completions/max_terminated_length": 302.2,
|
|
"completions/mean_length": 217.0875,
|
|
"completions/mean_terminated_length": 217.0875,
|
|
"completions/min_length": 138.9,
|
|
"completions/min_terminated_length": 138.9,
|
|
"entropy": 1.0599846214056015,
|
|
"epoch": 0.9984370690447734,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.12183108180761337,
|
|
"kl": 0.03809760370641015,
|
|
"learning_rate": 3.984324758159289e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 47246061.0,
|
|
"reward": -0.0875,
|
|
"reward_std": 0.26106746792793273,
|
|
"rewards/reward_fn/mean": -0.0875,
|
|
"rewards/reward_fn/std": 0.26106746792793273,
|
|
"sampling/importance_sampling_ratio/max": 1.3003975749015808,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999008655548096,
|
|
"sampling/importance_sampling_ratio/min": 0.7912079811096191,
|
|
"sampling/sampling_logp_difference/max": 0.2872904360294342,
|
|
"sampling/sampling_logp_difference/mean": 0.02050208393484354,
|
|
"step": 10860
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.1,
|
|
"completions/max_terminated_length": 316.1,
|
|
"completions/mean_length": 231.7875,
|
|
"completions/mean_terminated_length": 231.7875,
|
|
"completions/min_length": 152.4,
|
|
"completions/min_terminated_length": 152.4,
|
|
"entropy": 1.1013551726937294,
|
|
"epoch": 0.9993564401949067,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 5.377572059631348,
|
|
"kl": 0.030392793647479266,
|
|
"learning_rate": 3.98228527904902e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 47293596.0,
|
|
"reward": 0.05625,
|
|
"reward_std": 0.43449294567108154,
|
|
"rewards/reward_fn/mean": 0.05625,
|
|
"rewards/reward_fn/std": 0.4344929575920105,
|
|
"sampling/importance_sampling_ratio/max": 1.2969264030456542,
|
|
"sampling/importance_sampling_ratio/mean": 0.9999637007713318,
|
|
"sampling/importance_sampling_ratio/min": 0.7700069010257721,
|
|
"sampling/sampling_logp_difference/max": 0.29123281836509707,
|
|
"sampling/sampling_logp_difference/mean": 0.0211251026019454,
|
|
"step": 10870
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 32631,
|
|
"num_input_tokens_seen": 47321224,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|