10511 lines
405 KiB
JSON
10511 lines
405 KiB
JSON
[
|
|
{
|
|
"loss": 0.1827935427427292,
|
|
"grad_norm": 1.4597944021224976,
|
|
"learning_rate": 0.0,
|
|
"num_tokens": 11460.0,
|
|
"completions/mean_length": 746.375,
|
|
"completions/min_length": 172.0,
|
|
"completions/max_length": 968.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 458.3333435058594,
|
|
"completions/min_terminated_length": 172.0,
|
|
"completions/max_terminated_length": 968.0,
|
|
"tools/call_frequency": 16.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.12336309254169464,
|
|
"rewards/reward_func/std": 0.27320101857185364,
|
|
"reward": 0.12336309254169464,
|
|
"reward_std": 0.27320098876953125,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014739650068804622,
|
|
"sampling/sampling_logp_difference/max": 0.49289464950561523,
|
|
"sampling/importance_sampling_ratio/min": 0.6694592833518982,
|
|
"sampling/importance_sampling_ratio/mean": 0.9493762850761414,
|
|
"sampling/importance_sampling_ratio/max": 1.5817824602127075,
|
|
"kl": 0.0,
|
|
"entropy": 0.02768590796040371,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 15.11863349750638,
|
|
"epoch": 2.604166666666667e-05,
|
|
"step": 1
|
|
},
|
|
{
|
|
"loss": -0.261507123708725,
|
|
"grad_norm": 1.2871638536453247,
|
|
"learning_rate": 5e-08,
|
|
"num_tokens": 23596.0,
|
|
"completions/mean_length": 831.0,
|
|
"completions/min_length": 201.0,
|
|
"completions/max_length": 939.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 690.3333740234375,
|
|
"completions/min_terminated_length": 201.0,
|
|
"completions/max_terminated_length": 939.0,
|
|
"tools/call_frequency": 19.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0007969809230417013,
|
|
"sampling/sampling_logp_difference/max": 0.3981339931488037,
|
|
"sampling/importance_sampling_ratio/min": 0.6284723281860352,
|
|
"sampling/importance_sampling_ratio/mean": 0.8799340128898621,
|
|
"sampling/importance_sampling_ratio/max": 1.1058579683303833,
|
|
"kl": 0.0,
|
|
"entropy": 0.016428090282715857,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.451946565881371,
|
|
"epoch": 5.208333333333334e-05,
|
|
"step": 2
|
|
},
|
|
{
|
|
"loss": 0.13511055707931519,
|
|
"grad_norm": 1.725199580192566,
|
|
"learning_rate": 1e-07,
|
|
"num_tokens": 34324.0,
|
|
"completions/mean_length": 655.625,
|
|
"completions/min_length": 169.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 413.75,
|
|
"completions/min_terminated_length": 169.0,
|
|
"completions/max_terminated_length": 809.0,
|
|
"tools/call_frequency": 16.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21458333730697632,
|
|
"rewards/reward_func/std": 0.2785574495792389,
|
|
"reward": 0.21458333730697632,
|
|
"reward_std": 0.2785574197769165,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0024352525360882282,
|
|
"sampling/sampling_logp_difference/max": 0.6355695724487305,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.5688159465789795,
|
|
"sampling/importance_sampling_ratio/max": 0.8797624707221985,
|
|
"kl": 0.0002505501249743247,
|
|
"entropy": 0.04492050572298467,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.751723470166326,
|
|
"epoch": 7.8125e-05,
|
|
"step": 3
|
|
},
|
|
{
|
|
"loss": -0.016194790601730347,
|
|
"grad_norm": 3.2130491733551025,
|
|
"learning_rate": 1.5e-07,
|
|
"num_tokens": 43654.0,
|
|
"completions/mean_length": 480.75,
|
|
"completions/min_length": 154.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 336.0,
|
|
"completions/min_terminated_length": 154.0,
|
|
"completions/max_terminated_length": 884.0,
|
|
"tools/call_frequency": 12.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1137152761220932,
|
|
"rewards/reward_func/std": 0.2676820755004883,
|
|
"reward": 0.1137152761220932,
|
|
"reward_std": 0.2676820456981659,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.00220000883564353,
|
|
"sampling/sampling_logp_difference/max": 0.596961259841919,
|
|
"sampling/importance_sampling_ratio/min": 0.5027801394462585,
|
|
"sampling/importance_sampling_ratio/mean": 0.9069873690605164,
|
|
"sampling/importance_sampling_ratio/max": 1.669683575630188,
|
|
"kl": 0.00022831466372252862,
|
|
"entropy": 0.04863522437517531,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.665006207302213,
|
|
"epoch": 0.00010416666666666667,
|
|
"step": 4
|
|
},
|
|
{
|
|
"loss": -0.4495558440685272,
|
|
"grad_norm": 2.0488195419311523,
|
|
"learning_rate": 2e-07,
|
|
"num_tokens": 53131.0,
|
|
"completions/mean_length": 499.0,
|
|
"completions/min_length": 170.0,
|
|
"completions/max_length": 1023.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 424.14288330078125,
|
|
"completions/min_terminated_length": 170.0,
|
|
"completions/max_terminated_length": 946.0,
|
|
"tools/call_frequency": 11.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": -0.02500000037252903,
|
|
"rewards/reward_func/std": 0.08017837256193161,
|
|
"reward": -0.02500000037252903,
|
|
"reward_std": 0.08017837256193161,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002664719009771943,
|
|
"sampling/sampling_logp_difference/max": 0.359708309173584,
|
|
"sampling/importance_sampling_ratio/min": 0.45191431045532227,
|
|
"sampling/importance_sampling_ratio/mean": 0.8167451620101929,
|
|
"sampling/importance_sampling_ratio/max": 1.1800522804260254,
|
|
"kl": 0.00018751498600977357,
|
|
"entropy": 0.0560183891793713,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.975067878142,
|
|
"epoch": 0.00013020833333333333,
|
|
"step": 5
|
|
},
|
|
{
|
|
"loss": -0.5143288969993591,
|
|
"grad_norm": 2.862942934036255,
|
|
"learning_rate": 2.5e-07,
|
|
"num_tokens": 63850.0,
|
|
"completions/mean_length": 654.875,
|
|
"completions/min_length": 173.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 492.6000061035156,
|
|
"completions/min_terminated_length": 173.0,
|
|
"completions/max_terminated_length": 916.0,
|
|
"tools/call_frequency": 15.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": -0.0062500000931322575,
|
|
"rewards/reward_func/std": 0.07763238251209259,
|
|
"reward": -0.0062500000931322575,
|
|
"reward_std": 0.077632375061512,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015723961405456066,
|
|
"sampling/sampling_logp_difference/max": 0.6218194961547852,
|
|
"sampling/importance_sampling_ratio/min": 0.6238558888435364,
|
|
"sampling/importance_sampling_ratio/mean": 1.1268502473831177,
|
|
"sampling/importance_sampling_ratio/max": 2.118692398071289,
|
|
"kl": 0.00015170156422072978,
|
|
"entropy": 0.030710655904840678,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.164958016946912,
|
|
"epoch": 0.00015625,
|
|
"step": 6
|
|
},
|
|
{
|
|
"loss": -0.41568493843078613,
|
|
"grad_norm": 1.5634326934814453,
|
|
"learning_rate": 3e-07,
|
|
"num_tokens": 75193.0,
|
|
"completions/mean_length": 732.5,
|
|
"completions/min_length": 97.0,
|
|
"completions/max_length": 944.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 541.75,
|
|
"completions/min_terminated_length": 97.0,
|
|
"completions/max_terminated_length": 944.0,
|
|
"tools/call_frequency": 16.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.012500000186264515,
|
|
"rewards/reward_func/std": 0.06943651288747787,
|
|
"reward": 0.012500000186264515,
|
|
"reward_std": 0.06943650543689728,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014808306004852057,
|
|
"sampling/sampling_logp_difference/max": 0.5288983583450317,
|
|
"sampling/importance_sampling_ratio/min": 0.4345861077308655,
|
|
"sampling/importance_sampling_ratio/mean": 1.0106513500213623,
|
|
"sampling/importance_sampling_ratio/max": 1.3749502897262573,
|
|
"kl": 0.0001614846000848047,
|
|
"entropy": 0.02566744282376021,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.274704342707992,
|
|
"epoch": 0.00018229166666666667,
|
|
"step": 7
|
|
},
|
|
{
|
|
"loss": -0.1888810247182846,
|
|
"grad_norm": 1.2660902738571167,
|
|
"learning_rate": 3.5e-07,
|
|
"num_tokens": 87487.0,
|
|
"completions/mean_length": 851.125,
|
|
"completions/min_length": 167.0,
|
|
"completions/max_length": 986.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 693.0,
|
|
"completions/min_terminated_length": 167.0,
|
|
"completions/max_terminated_length": 963.0,
|
|
"tools/call_frequency": 18.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.0546875,
|
|
"rewards/reward_func/std": 0.09061729907989502,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.09061729907989502,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015309966402128339,
|
|
"sampling/sampling_logp_difference/max": 1.4471790790557861,
|
|
"sampling/importance_sampling_ratio/min": 0.14186790585517883,
|
|
"sampling/importance_sampling_ratio/mean": 0.6313877105712891,
|
|
"sampling/importance_sampling_ratio/max": 0.9336322546005249,
|
|
"kl": 0.00011950841030738957,
|
|
"entropy": 0.016202514700125903,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.842062007635832,
|
|
"epoch": 0.00020833333333333335,
|
|
"step": 8
|
|
},
|
|
{
|
|
"loss": 0.053002066910266876,
|
|
"grad_norm": 2.5570223331451416,
|
|
"learning_rate": 4e-07,
|
|
"num_tokens": 96889.0,
|
|
"completions/mean_length": 488.875,
|
|
"completions/min_length": 178.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 434.0000305175781,
|
|
"completions/min_terminated_length": 178.0,
|
|
"completions/max_terminated_length": 943.0,
|
|
"tools/call_frequency": 12.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.11354167759418488,
|
|
"rewards/reward_func/std": 0.28729507327079773,
|
|
"reward": 0.11354167759418488,
|
|
"reward_std": 0.28729504346847534,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0030449291225522757,
|
|
"sampling/sampling_logp_difference/max": 0.6426939964294434,
|
|
"sampling/importance_sampling_ratio/min": 0.39020252227783203,
|
|
"sampling/importance_sampling_ratio/mean": 0.7954643964767456,
|
|
"sampling/importance_sampling_ratio/max": 1.3060299158096313,
|
|
"kl": 0.0006131824329713709,
|
|
"entropy": 0.05258491728454828,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.919184625148773,
|
|
"epoch": 0.000234375,
|
|
"step": 9
|
|
},
|
|
{
|
|
"loss": -0.07246054708957672,
|
|
"grad_norm": 1.9255688190460205,
|
|
"learning_rate": 4.5e-07,
|
|
"num_tokens": 108205.0,
|
|
"completions/mean_length": 729.125,
|
|
"completions/min_length": 157.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 544.5,
|
|
"completions/min_terminated_length": 157.0,
|
|
"completions/max_terminated_length": 934.0,
|
|
"tools/call_frequency": 17.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06577380746603012,
|
|
"rewards/reward_func/std": 0.17914392054080963,
|
|
"reward": 0.06577380746603012,
|
|
"reward_std": 0.17914393544197083,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0012505335034802556,
|
|
"sampling/sampling_logp_difference/max": 0.6249988079071045,
|
|
"sampling/importance_sampling_ratio/min": 0.43585535883903503,
|
|
"sampling/importance_sampling_ratio/mean": 1.0506987571716309,
|
|
"sampling/importance_sampling_ratio/max": 2.2679803371429443,
|
|
"kl": 0.00020860102131337044,
|
|
"entropy": 0.025938350532669574,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.819276506081223,
|
|
"epoch": 0.00026041666666666666,
|
|
"step": 10
|
|
},
|
|
{
|
|
"loss": 0.013630369678139687,
|
|
"grad_norm": 2.448845863342285,
|
|
"learning_rate": 5e-07,
|
|
"num_tokens": 118288.0,
|
|
"completions/mean_length": 575.125,
|
|
"completions/min_length": 144.0,
|
|
"completions/max_length": 986.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 355.20001220703125,
|
|
"completions/min_terminated_length": 144.0,
|
|
"completions/max_terminated_length": 924.0,
|
|
"tools/call_frequency": 13.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.04965277388691902,
|
|
"rewards/reward_func/std": 0.19548721611499786,
|
|
"reward": 0.04965277388691902,
|
|
"reward_std": 0.19548721611499786,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0025224804412573576,
|
|
"sampling/sampling_logp_difference/max": 0.9216582775115967,
|
|
"sampling/importance_sampling_ratio/min": 0.2349359095096588,
|
|
"sampling/importance_sampling_ratio/mean": 0.9609678387641907,
|
|
"sampling/importance_sampling_ratio/max": 2.0699515342712402,
|
|
"kl": 0.0004645602675736882,
|
|
"entropy": 0.04753242793958634,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.538785552605987,
|
|
"epoch": 0.00028645833333333333,
|
|
"step": 11
|
|
},
|
|
{
|
|
"loss": -0.27833986282348633,
|
|
"grad_norm": 1.3661935329437256,
|
|
"learning_rate": 4.982758620689655e-07,
|
|
"num_tokens": 129660.0,
|
|
"completions/mean_length": 735.75,
|
|
"completions/min_length": 142.0,
|
|
"completions/max_length": 937.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 425.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"completions/max_terminated_length": 907.0,
|
|
"tools/call_frequency": 16.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05312500521540642,
|
|
"rewards/reward_func/std": 0.14665162563323975,
|
|
"reward": 0.05312500521540642,
|
|
"reward_std": 0.14665161073207855,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0011708264937624335,
|
|
"sampling/sampling_logp_difference/max": 0.33389294147491455,
|
|
"sampling/importance_sampling_ratio/min": 0.5948313474655151,
|
|
"sampling/importance_sampling_ratio/mean": 1.0551820993423462,
|
|
"sampling/importance_sampling_ratio/max": 1.9267381429672241,
|
|
"kl": 6.204826445355138e-05,
|
|
"entropy": 0.018247017986141145,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.184796966612339,
|
|
"epoch": 0.0003125,
|
|
"step": 12
|
|
},
|
|
{
|
|
"loss": -0.2948645353317261,
|
|
"grad_norm": 3.541626214981079,
|
|
"learning_rate": 4.96551724137931e-07,
|
|
"num_tokens": 140141.0,
|
|
"completions/mean_length": 624.0,
|
|
"completions/min_length": 142.0,
|
|
"completions/max_length": 991.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 465.6000061035156,
|
|
"completions/min_terminated_length": 142.0,
|
|
"completions/max_terminated_length": 991.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.0677083283662796,
|
|
"rewards/reward_func/std": 0.24350440502166748,
|
|
"reward": 0.0677083283662796,
|
|
"reward_std": 0.24350440502166748,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017545621376484632,
|
|
"sampling/sampling_logp_difference/max": 0.48372888565063477,
|
|
"sampling/importance_sampling_ratio/min": 0.1752922683954239,
|
|
"sampling/importance_sampling_ratio/mean": 1.238909125328064,
|
|
"sampling/importance_sampling_ratio/max": 2.7641284465789795,
|
|
"kl": 0.00014763017588848015,
|
|
"entropy": 0.018221410282421857,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.5901396330446,
|
|
"epoch": 0.0003385416666666667,
|
|
"step": 13
|
|
},
|
|
{
|
|
"loss": 0.22289058566093445,
|
|
"grad_norm": 0.8254400491714478,
|
|
"learning_rate": 4.948275862068965e-07,
|
|
"num_tokens": 151712.0,
|
|
"completions/mean_length": 760.5,
|
|
"completions/min_length": 276.0,
|
|
"completions/max_length": 938.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 489.3333435058594,
|
|
"completions/min_terminated_length": 276.0,
|
|
"completions/max_terminated_length": 915.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2176339328289032,
|
|
"rewards/reward_func/std": 0.31572380661964417,
|
|
"reward": 0.2176339328289032,
|
|
"reward_std": 0.3157237768173218,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017725247889757156,
|
|
"sampling/sampling_logp_difference/max": 0.43998420238494873,
|
|
"sampling/importance_sampling_ratio/min": 0.2768198847770691,
|
|
"sampling/importance_sampling_ratio/mean": 0.510668158531189,
|
|
"sampling/importance_sampling_ratio/max": 0.9676974415779114,
|
|
"kl": 0.00015959835792500598,
|
|
"entropy": 0.033386378665454686,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.534008683636785,
|
|
"epoch": 0.00036458333333333335,
|
|
"step": 14
|
|
},
|
|
{
|
|
"loss": 0.17401374876499176,
|
|
"grad_norm": 3.161911725997925,
|
|
"learning_rate": 4.93103448275862e-07,
|
|
"num_tokens": 162495.0,
|
|
"completions/mean_length": 662.875,
|
|
"completions/min_length": 205.0,
|
|
"completions/max_length": 957.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 403.5,
|
|
"completions/min_terminated_length": 205.0,
|
|
"completions/max_terminated_length": 927.0,
|
|
"tools/call_frequency": 14.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.15156249701976776,
|
|
"rewards/reward_func/std": 0.2541123926639557,
|
|
"reward": 0.15156249701976776,
|
|
"reward_std": 0.2541123628616333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022795340046286583,
|
|
"sampling/sampling_logp_difference/max": 1.035105586051941,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.643468976020813,
|
|
"sampling/importance_sampling_ratio/max": 1.1692376136779785,
|
|
"kl": 0.0004783355934705469,
|
|
"entropy": 0.03411270503420383,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.328358532860875,
|
|
"epoch": 0.000390625,
|
|
"step": 15
|
|
},
|
|
{
|
|
"loss": -0.23745884001255035,
|
|
"grad_norm": 0.8878580331802368,
|
|
"learning_rate": 4.913793103448275e-07,
|
|
"num_tokens": 174547.0,
|
|
"completions/mean_length": 821.5,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 978.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 713.5,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 915.0,
|
|
"tools/call_frequency": 20.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0008860031375661492,
|
|
"sampling/sampling_logp_difference/max": 0.5278483629226685,
|
|
"sampling/importance_sampling_ratio/min": 0.33447009325027466,
|
|
"sampling/importance_sampling_ratio/mean": 0.8628897666931152,
|
|
"sampling/importance_sampling_ratio/max": 1.2707186937332153,
|
|
"kl": 6.335315390515461e-05,
|
|
"entropy": 0.00875668102526106,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.524089926853776,
|
|
"epoch": 0.0004166666666666667,
|
|
"step": 16
|
|
},
|
|
{
|
|
"loss": -0.3115900456905365,
|
|
"grad_norm": 4.0940985679626465,
|
|
"learning_rate": 4.89655172413793e-07,
|
|
"num_tokens": 185855.0,
|
|
"completions/mean_length": 728.625,
|
|
"completions/min_length": 168.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 545.5,
|
|
"completions/min_terminated_length": 168.0,
|
|
"completions/max_terminated_length": 922.0,
|
|
"tools/call_frequency": 17.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.012500000186264515,
|
|
"rewards/reward_func/std": 0.06943651288747787,
|
|
"reward": 0.012500000186264515,
|
|
"reward_std": 0.06943650543689728,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017367153195664287,
|
|
"sampling/sampling_logp_difference/max": 0.8774404525756836,
|
|
"sampling/importance_sampling_ratio/min": 0.4356679916381836,
|
|
"sampling/importance_sampling_ratio/mean": 1.1706042289733887,
|
|
"sampling/importance_sampling_ratio/max": 2.5660619735717773,
|
|
"kl": 0.000309262212112138,
|
|
"entropy": 0.029268034413689747,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.736797543242574,
|
|
"epoch": 0.0004427083333333333,
|
|
"step": 17
|
|
},
|
|
{
|
|
"loss": -0.0035714544355869293,
|
|
"grad_norm": 2.5381369590759277,
|
|
"learning_rate": 4.879310344827585e-07,
|
|
"num_tokens": 195081.0,
|
|
"completions/mean_length": 468.125,
|
|
"completions/min_length": 84.0,
|
|
"completions/max_length": 895.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/mean_terminated_length": 468.125,
|
|
"completions/min_terminated_length": 84.0,
|
|
"completions/max_terminated_length": 895.0,
|
|
"tools/call_frequency": 12.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2671875059604645,
|
|
"rewards/reward_func/std": 0.32900792360305786,
|
|
"reward": 0.2671875059604645,
|
|
"reward_std": 0.32900792360305786,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002830307697877288,
|
|
"sampling/sampling_logp_difference/max": 0.9735250473022461,
|
|
"sampling/importance_sampling_ratio/min": 0.38942375779151917,
|
|
"sampling/importance_sampling_ratio/mean": 0.8064284920692444,
|
|
"sampling/importance_sampling_ratio/max": 1.2885123491287231,
|
|
"kl": 0.0005776227346814267,
|
|
"entropy": 0.055245712923351675,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.01355754584074,
|
|
"epoch": 0.00046875,
|
|
"step": 18
|
|
},
|
|
{
|
|
"loss": 0.08326569199562073,
|
|
"grad_norm": 3.511403799057007,
|
|
"learning_rate": 4.86206896551724e-07,
|
|
"num_tokens": 205833.0,
|
|
"completions/mean_length": 659.625,
|
|
"completions/min_length": 172.0,
|
|
"completions/max_length": 956.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 499.6000061035156,
|
|
"completions/min_terminated_length": 172.0,
|
|
"completions/max_terminated_length": 929.0,
|
|
"tools/call_frequency": 15.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.08437499403953552,
|
|
"rewards/reward_func/std": 0.22871437668800354,
|
|
"reward": 0.08437499403953552,
|
|
"reward_std": 0.22871437668800354,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001922936993651092,
|
|
"sampling/sampling_logp_difference/max": 0.7091507911682129,
|
|
"sampling/importance_sampling_ratio/min": 0.3113582134246826,
|
|
"sampling/importance_sampling_ratio/mean": 1.0483355522155762,
|
|
"sampling/importance_sampling_ratio/max": 1.403739333152771,
|
|
"kl": 0.00037196421112639655,
|
|
"entropy": 0.034325361251831055,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.279798299074173,
|
|
"epoch": 0.0004947916666666667,
|
|
"step": 19
|
|
},
|
|
{
|
|
"loss": -0.012817198410630226,
|
|
"grad_norm": 1.0862401723861694,
|
|
"learning_rate": 4.844827586206897e-07,
|
|
"num_tokens": 217931.0,
|
|
"completions/mean_length": 826.75,
|
|
"completions/min_length": 193.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 762.6000366210938,
|
|
"completions/min_terminated_length": 193.0,
|
|
"completions/max_terminated_length": 944.0,
|
|
"tools/call_frequency": 19.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10208333283662796,
|
|
"rewards/reward_func/std": 0.2144460529088974,
|
|
"reward": 0.10208333283662796,
|
|
"reward_std": 0.2144460380077362,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0012041025329381227,
|
|
"sampling/sampling_logp_difference/max": 0.5278481245040894,
|
|
"sampling/importance_sampling_ratio/min": 0.24310140311717987,
|
|
"sampling/importance_sampling_ratio/mean": 0.9718503952026367,
|
|
"sampling/importance_sampling_ratio/max": 1.7723968029022217,
|
|
"kl": 0.00023671223561905208,
|
|
"entropy": 0.014663240173831582,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.887908939272165,
|
|
"epoch": 0.0005208333333333333,
|
|
"step": 20
|
|
},
|
|
{
|
|
"loss": -0.47885897755622864,
|
|
"grad_norm": 2.764204502105713,
|
|
"learning_rate": 4.827586206896552e-07,
|
|
"num_tokens": 228496.0,
|
|
"completions/mean_length": 635.75,
|
|
"completions/min_length": 139.0,
|
|
"completions/max_length": 916.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 371.75,
|
|
"completions/min_terminated_length": 139.0,
|
|
"completions/max_terminated_length": 900.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1770833283662796,
|
|
"rewards/reward_func/std": 0.31467893719673157,
|
|
"reward": 0.1770833283662796,
|
|
"reward_std": 0.3146789073944092,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014256216818466783,
|
|
"sampling/sampling_logp_difference/max": 0.4963933825492859,
|
|
"sampling/importance_sampling_ratio/min": 0.5183834433555603,
|
|
"sampling/importance_sampling_ratio/mean": 0.9836266040802002,
|
|
"sampling/importance_sampling_ratio/max": 1.584794521331787,
|
|
"kl": 0.0003084373099682125,
|
|
"entropy": 0.029118798265699297,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.617893446236849,
|
|
"epoch": 0.000546875,
|
|
"step": 21
|
|
},
|
|
{
|
|
"loss": -0.06596201658248901,
|
|
"grad_norm": 1.3604154586791992,
|
|
"learning_rate": 4.810344827586207e-07,
|
|
"num_tokens": 239837.0,
|
|
"completions/mean_length": 732.75,
|
|
"completions/min_length": 196.0,
|
|
"completions/max_length": 915.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 458.3333435058594,
|
|
"completions/min_terminated_length": 196.0,
|
|
"completions/max_terminated_length": 914.0,
|
|
"tools/call_frequency": 17.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17152777314186096,
|
|
"rewards/reward_func/std": 0.2828378677368164,
|
|
"reward": 0.17152777314186096,
|
|
"reward_std": 0.282837837934494,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014051601756364107,
|
|
"sampling/sampling_logp_difference/max": 0.43248438835144043,
|
|
"sampling/importance_sampling_ratio/min": 0.5559346079826355,
|
|
"sampling/importance_sampling_ratio/mean": 0.8648233413696289,
|
|
"sampling/importance_sampling_ratio/max": 1.463281273841858,
|
|
"kl": 0.00019491067450871924,
|
|
"entropy": 0.029180668701883405,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.200672704726458,
|
|
"epoch": 0.0005729166666666667,
|
|
"step": 22
|
|
},
|
|
{
|
|
"loss": 0.11418360471725464,
|
|
"grad_norm": 1.6365529298782349,
|
|
"learning_rate": 4.793103448275862e-07,
|
|
"num_tokens": 251312.0,
|
|
"completions/mean_length": 749.0,
|
|
"completions/min_length": 86.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 645.2000122070312,
|
|
"completions/min_terminated_length": 86.0,
|
|
"completions/max_terminated_length": 941.0,
|
|
"tools/call_frequency": 17.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.08750000596046448,
|
|
"rewards/reward_func/std": 0.17474472522735596,
|
|
"reward": 0.08750000596046448,
|
|
"reward_std": 0.17474469542503357,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016364507609978318,
|
|
"sampling/sampling_logp_difference/max": 0.452467679977417,
|
|
"sampling/importance_sampling_ratio/min": 0.3670050799846649,
|
|
"sampling/importance_sampling_ratio/mean": 0.9000353813171387,
|
|
"sampling/importance_sampling_ratio/max": 1.7929006814956665,
|
|
"kl": 0.00048575789625715515,
|
|
"entropy": 0.03412623889744282,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.978007948026061,
|
|
"epoch": 0.0005989583333333333,
|
|
"step": 23
|
|
},
|
|
{
|
|
"loss": -0.5659245252609253,
|
|
"grad_norm": 3.199495315551758,
|
|
"learning_rate": 4.775862068965517e-07,
|
|
"num_tokens": 262950.0,
|
|
"completions/mean_length": 769.375,
|
|
"completions/min_length": 250.0,
|
|
"completions/max_length": 977.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 488.3333435058594,
|
|
"completions/min_terminated_length": 250.0,
|
|
"completions/max_terminated_length": 946.0,
|
|
"tools/call_frequency": 16.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.0677083283662796,
|
|
"rewards/reward_func/std": 0.09578803926706314,
|
|
"reward": 0.0677083283662796,
|
|
"reward_std": 0.09578803181648254,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014710350660607219,
|
|
"sampling/sampling_logp_difference/max": 1.4230310916900635,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.9081178307533264,
|
|
"sampling/importance_sampling_ratio/max": 1.9899202585220337,
|
|
"kl": 0.0003319300122370805,
|
|
"entropy": 0.019700150587596,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.312813334167004,
|
|
"epoch": 0.000625,
|
|
"step": 24
|
|
},
|
|
{
|
|
"loss": -0.0447189100086689,
|
|
"grad_norm": 1.7205936908721924,
|
|
"learning_rate": 4.7586206896551725e-07,
|
|
"num_tokens": 274490.0,
|
|
"completions/mean_length": 757.625,
|
|
"completions/min_length": 215.0,
|
|
"completions/max_length": 925.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 671.7999877929688,
|
|
"completions/min_terminated_length": 215.0,
|
|
"completions/max_terminated_length": 925.0,
|
|
"tools/call_frequency": 18.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.04843750223517418,
|
|
"rewards/reward_func/std": 0.07689205557107925,
|
|
"reward": 0.04843750223517418,
|
|
"reward_std": 0.07689205557107925,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001273148925974965,
|
|
"sampling/sampling_logp_difference/max": 0.26259955763816833,
|
|
"sampling/importance_sampling_ratio/min": 0.5838127136230469,
|
|
"sampling/importance_sampling_ratio/mean": 0.847714900970459,
|
|
"sampling/importance_sampling_ratio/max": 1.003871202468872,
|
|
"kl": 0.00015287923753248833,
|
|
"entropy": 0.035843983641825616,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.774942817166448,
|
|
"epoch": 0.0006510416666666666,
|
|
"step": 25
|
|
},
|
|
{
|
|
"loss": -0.045191336423158646,
|
|
"grad_norm": 1.0773961544036865,
|
|
"learning_rate": 4.7413793103448276e-07,
|
|
"num_tokens": 287106.0,
|
|
"completions/mean_length": 892.75,
|
|
"completions/min_length": 852.0,
|
|
"completions/max_length": 913.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 889.0,
|
|
"completions/min_terminated_length": 880.0,
|
|
"completions/max_terminated_length": 898.0,
|
|
"tools/call_frequency": 22.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10625000298023224,
|
|
"rewards/reward_func/std": 0.1590990424156189,
|
|
"reward": 0.10625000298023224,
|
|
"reward_std": 0.1590990275144577,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0008657958824187517,
|
|
"sampling/sampling_logp_difference/max": 0.32401013374328613,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8601902723312378,
|
|
"sampling/importance_sampling_ratio/max": 1.1449334621429443,
|
|
"kl": 0.000127534308589361,
|
|
"entropy": 0.010987800429575145,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.163068493828177,
|
|
"epoch": 0.0006770833333333334,
|
|
"step": 26
|
|
},
|
|
{
|
|
"loss": 0.3643893897533417,
|
|
"grad_norm": 6.705402851104736,
|
|
"learning_rate": 4.7241379310344827e-07,
|
|
"num_tokens": 297136.0,
|
|
"completions/mean_length": 569.375,
|
|
"completions/min_length": 197.0,
|
|
"completions/max_length": 936.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 453.8333435058594,
|
|
"completions/min_terminated_length": 197.0,
|
|
"completions/max_terminated_length": 936.0,
|
|
"tools/call_frequency": 13.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2657986283302307,
|
|
"rewards/reward_func/std": 0.2913707196712494,
|
|
"reward": 0.2657986283302307,
|
|
"reward_std": 0.2913707494735718,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002442025812342763,
|
|
"sampling/sampling_logp_difference/max": 0.7211463451385498,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.1092699766159058,
|
|
"sampling/importance_sampling_ratio/max": 2.965717077255249,
|
|
"kl": 0.0020042841966869673,
|
|
"entropy": 0.04371687467209995,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.418146455660462,
|
|
"epoch": 0.000703125,
|
|
"step": 27
|
|
},
|
|
{
|
|
"loss": -0.8481144905090332,
|
|
"grad_norm": 6.174228668212891,
|
|
"learning_rate": 4.706896551724138e-07,
|
|
"num_tokens": 306896.0,
|
|
"completions/mean_length": 534.75,
|
|
"completions/min_length": 141.0,
|
|
"completions/max_length": 946.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 413.3333435058594,
|
|
"completions/min_terminated_length": 141.0,
|
|
"completions/max_terminated_length": 946.0,
|
|
"tools/call_frequency": 12.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": -0.02500000037252903,
|
|
"rewards/reward_func/std": 0.08017837256193161,
|
|
"reward": -0.02500000037252903,
|
|
"reward_std": 0.08017837256193161,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0021727937273681164,
|
|
"sampling/sampling_logp_difference/max": 1.4230237007141113,
|
|
"sampling/importance_sampling_ratio/min": 0.24237510561943054,
|
|
"sampling/importance_sampling_ratio/mean": 1.031205654144287,
|
|
"sampling/importance_sampling_ratio/max": 2.1671206951141357,
|
|
"kl": 0.0032036943446200894,
|
|
"entropy": 0.037270010681822896,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.267284324392676,
|
|
"epoch": 0.0007291666666666667,
|
|
"step": 28
|
|
},
|
|
{
|
|
"loss": 0.24251770973205566,
|
|
"grad_norm": 1.9181212186813354,
|
|
"learning_rate": 4.689655172413793e-07,
|
|
"num_tokens": 317778.0,
|
|
"completions/mean_length": 675.75,
|
|
"completions/min_length": 147.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 636.2857666015625,
|
|
"completions/min_terminated_length": 147.0,
|
|
"completions/max_terminated_length": 935.0,
|
|
"tools/call_frequency": 15.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1718749850988388,
|
|
"rewards/reward_func/std": 0.3166333734989166,
|
|
"reward": 0.1718749850988388,
|
|
"reward_std": 0.31663334369659424,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001911571598611772,
|
|
"sampling/sampling_logp_difference/max": 1.0281217098236084,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8459687232971191,
|
|
"sampling/importance_sampling_ratio/max": 1.2212952375411987,
|
|
"kl": 0.0009548202418727669,
|
|
"entropy": 0.039005081984214485,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.229724664241076,
|
|
"epoch": 0.0007552083333333333,
|
|
"step": 29
|
|
},
|
|
{
|
|
"loss": -0.283731073141098,
|
|
"grad_norm": 2.3733885288238525,
|
|
"learning_rate": 4.672413793103448e-07,
|
|
"num_tokens": 330030.0,
|
|
"completions/mean_length": 845.375,
|
|
"completions/min_length": 235.0,
|
|
"completions/max_length": 1003.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 782.6000366210938,
|
|
"completions/min_terminated_length": 235.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 19.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05000000074505806,
|
|
"rewards/reward_func/std": 0.08017837256193161,
|
|
"reward": 0.05000000074505806,
|
|
"reward_std": 0.08017837256193161,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0011006047716364264,
|
|
"sampling/sampling_logp_difference/max": 0.43122565746307373,
|
|
"sampling/importance_sampling_ratio/min": 0.4104643166065216,
|
|
"sampling/importance_sampling_ratio/mean": 0.8575743436813354,
|
|
"sampling/importance_sampling_ratio/max": 1.4309362173080444,
|
|
"kl": 0.00021837840978378154,
|
|
"entropy": 0.014480334328254685,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.646228479221463,
|
|
"epoch": 0.00078125,
|
|
"step": 30
|
|
},
|
|
{
|
|
"loss": -0.23653531074523926,
|
|
"grad_norm": 0.840833306312561,
|
|
"learning_rate": 4.655172413793103e-07,
|
|
"num_tokens": 342177.0,
|
|
"completions/mean_length": 834.25,
|
|
"completions/min_length": 225.0,
|
|
"completions/max_length": 968.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 742.5,
|
|
"completions/min_terminated_length": 225.0,
|
|
"completions/max_terminated_length": 917.0,
|
|
"tools/call_frequency": 19.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0010216847294941545,
|
|
"sampling/sampling_logp_difference/max": 0.4507479667663574,
|
|
"sampling/importance_sampling_ratio/min": 0.3365154564380646,
|
|
"sampling/importance_sampling_ratio/mean": 0.9169121980667114,
|
|
"sampling/importance_sampling_ratio/max": 2.054715156555176,
|
|
"kl": 0.00020786568188668753,
|
|
"entropy": 0.016975120233837515,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.549037532880902,
|
|
"epoch": 0.0008072916666666667,
|
|
"step": 31
|
|
},
|
|
{
|
|
"loss": -0.13676826655864716,
|
|
"grad_norm": 4.244840145111084,
|
|
"learning_rate": 4.637931034482758e-07,
|
|
"num_tokens": 352859.0,
|
|
"completions/mean_length": 650.375,
|
|
"completions/min_length": 205.0,
|
|
"completions/max_length": 919.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 226.33334350585938,
|
|
"completions/min_terminated_length": 205.0,
|
|
"completions/max_terminated_length": 265.0,
|
|
"tools/call_frequency": 15.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23281249403953552,
|
|
"rewards/reward_func/std": 0.34942156076431274,
|
|
"reward": 0.23281249403953552,
|
|
"reward_std": 0.34942156076431274,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001932144514285028,
|
|
"sampling/sampling_logp_difference/max": 0.4955158233642578,
|
|
"sampling/importance_sampling_ratio/min": 0.37314677238464355,
|
|
"sampling/importance_sampling_ratio/mean": 0.9607480764389038,
|
|
"sampling/importance_sampling_ratio/max": 1.625891089439392,
|
|
"kl": 0.0018811069076036802,
|
|
"entropy": 0.03802944900235161,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.86856815777719,
|
|
"epoch": 0.0008333333333333334,
|
|
"step": 32
|
|
},
|
|
{
|
|
"loss": 0.35429295897483826,
|
|
"grad_norm": 5.527848243713379,
|
|
"learning_rate": 4.620689655172413e-07,
|
|
"num_tokens": 363062.0,
|
|
"completions/mean_length": 590.625,
|
|
"completions/min_length": 171.0,
|
|
"completions/max_length": 960.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 479.8333435058594,
|
|
"completions/min_terminated_length": 171.0,
|
|
"completions/max_terminated_length": 960.0,
|
|
"tools/call_frequency": 13.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.20146781206130981,
|
|
"rewards/reward_func/std": 0.2817278802394867,
|
|
"reward": 0.20146781206130981,
|
|
"reward_std": 0.2817278504371643,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0024167103692889214,
|
|
"sampling/sampling_logp_difference/max": 0.33373093605041504,
|
|
"sampling/importance_sampling_ratio/min": 0.3488825857639313,
|
|
"sampling/importance_sampling_ratio/mean": 0.8182227611541748,
|
|
"sampling/importance_sampling_ratio/max": 1.5994596481323242,
|
|
"kl": 0.002686779710529663,
|
|
"entropy": 0.035691179684363306,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.477436440065503,
|
|
"epoch": 0.000859375,
|
|
"step": 33
|
|
},
|
|
{
|
|
"loss": 0.15044048428535461,
|
|
"grad_norm": 1.3960310220718384,
|
|
"learning_rate": 4.603448275862069e-07,
|
|
"num_tokens": 374540.0,
|
|
"completions/mean_length": 750.75,
|
|
"completions/min_length": 208.0,
|
|
"completions/max_length": 955.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 455.66668701171875,
|
|
"completions/min_terminated_length": 208.0,
|
|
"completions/max_terminated_length": 916.0,
|
|
"tools/call_frequency": 16.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17604167759418488,
|
|
"rewards/reward_func/std": 0.2926831841468811,
|
|
"reward": 0.17604167759418488,
|
|
"reward_std": 0.2926831841468811,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016833762638270855,
|
|
"sampling/sampling_logp_difference/max": 0.4825162887573242,
|
|
"sampling/importance_sampling_ratio/min": 0.6056614518165588,
|
|
"sampling/importance_sampling_ratio/mean": 1.0012564659118652,
|
|
"sampling/importance_sampling_ratio/max": 1.461897373199463,
|
|
"kl": 0.0007303784605028341,
|
|
"entropy": 0.027316820400301367,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.145911324769258,
|
|
"epoch": 0.0008854166666666666,
|
|
"step": 34
|
|
},
|
|
{
|
|
"loss": 0.23567470908164978,
|
|
"grad_norm": 0.8548635840415955,
|
|
"learning_rate": 4.586206896551724e-07,
|
|
"num_tokens": 387214.0,
|
|
"completions/mean_length": 900.5,
|
|
"completions/min_length": 866.0,
|
|
"completions/max_length": 927.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 896.5,
|
|
"completions/min_terminated_length": 866.0,
|
|
"completions/max_terminated_length": 927.0,
|
|
"tools/call_frequency": 21.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16354167461395264,
|
|
"rewards/reward_func/std": 0.2433210164308548,
|
|
"reward": 0.16354167461395264,
|
|
"reward_std": 0.2433210015296936,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0012453746749088168,
|
|
"sampling/sampling_logp_difference/max": 0.6686383485794067,
|
|
"sampling/importance_sampling_ratio/min": 0.23233215510845184,
|
|
"sampling/importance_sampling_ratio/mean": 0.8257856369018555,
|
|
"sampling/importance_sampling_ratio/max": 1.344627857208252,
|
|
"kl": 0.0004161391786965396,
|
|
"entropy": 0.012758995464537293,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.227418741211295,
|
|
"epoch": 0.0009114583333333333,
|
|
"step": 35
|
|
},
|
|
{
|
|
"loss": 0.16829267144203186,
|
|
"grad_norm": 2.657780408859253,
|
|
"learning_rate": 4.5689655172413795e-07,
|
|
"num_tokens": 395873.0,
|
|
"completions/mean_length": 397.75,
|
|
"completions/min_length": 170.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 326.4285888671875,
|
|
"completions/min_terminated_length": 170.0,
|
|
"completions/max_terminated_length": 929.0,
|
|
"tools/call_frequency": 10.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.0664682537317276,
|
|
"rewards/reward_func/std": 0.23243336379528046,
|
|
"reward": 0.0664682537317276,
|
|
"reward_std": 0.23243334889411926,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003178231418132782,
|
|
"sampling/sampling_logp_difference/max": 0.6782898902893066,
|
|
"sampling/importance_sampling_ratio/min": 0.16409717500209808,
|
|
"sampling/importance_sampling_ratio/mean": 0.7944311499595642,
|
|
"sampling/importance_sampling_ratio/max": 2.808283805847168,
|
|
"kl": 0.0024459279184156912,
|
|
"entropy": 0.04908289958257228,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.810299593955278,
|
|
"epoch": 0.0009375,
|
|
"step": 36
|
|
},
|
|
{
|
|
"loss": 0.1114797443151474,
|
|
"grad_norm": 1.8561468124389648,
|
|
"learning_rate": 4.5517241379310346e-07,
|
|
"num_tokens": 406651.0,
|
|
"completions/mean_length": 662.375,
|
|
"completions/min_length": 226.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 424.0,
|
|
"completions/min_terminated_length": 226.0,
|
|
"completions/max_terminated_length": 881.0,
|
|
"tools/call_frequency": 16.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.33645832538604736,
|
|
"rewards/reward_func/std": 0.2596142292022705,
|
|
"reward": 0.33645832538604736,
|
|
"reward_std": 0.2596142292022705,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019980648066848516,
|
|
"sampling/sampling_logp_difference/max": 0.5238361358642578,
|
|
"sampling/importance_sampling_ratio/min": 0.2508929669857025,
|
|
"sampling/importance_sampling_ratio/mean": 1.142216682434082,
|
|
"sampling/importance_sampling_ratio/max": 2.6566147804260254,
|
|
"kl": 0.001080105702385481,
|
|
"entropy": 0.03969227778725326,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.54289066977799,
|
|
"epoch": 0.0009635416666666667,
|
|
"step": 37
|
|
},
|
|
{
|
|
"loss": 0.39239302277565,
|
|
"grad_norm": 2.764214038848877,
|
|
"learning_rate": 4.5344827586206896e-07,
|
|
"num_tokens": 416906.0,
|
|
"completions/mean_length": 596.25,
|
|
"completions/min_length": 236.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 487.16668701171875,
|
|
"completions/min_terminated_length": 236.0,
|
|
"completions/max_terminated_length": 934.0,
|
|
"tools/call_frequency": 13.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3515625,
|
|
"rewards/reward_func/std": 0.26830723881721497,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.26830726861953735,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002603382570669055,
|
|
"sampling/sampling_logp_difference/max": 0.6216655969619751,
|
|
"sampling/importance_sampling_ratio/min": 0.5178728699684143,
|
|
"sampling/importance_sampling_ratio/mean": 1.096362590789795,
|
|
"sampling/importance_sampling_ratio/max": 2.123946189880371,
|
|
"kl": 0.002432322991808178,
|
|
"entropy": 0.04524256428703666,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.617603242397308,
|
|
"epoch": 0.0009895833333333334,
|
|
"step": 38
|
|
},
|
|
{
|
|
"loss": -0.3050635755062103,
|
|
"grad_norm": 3.019944906234741,
|
|
"learning_rate": 4.5172413793103447e-07,
|
|
"num_tokens": 427533.0,
|
|
"completions/mean_length": 644.25,
|
|
"completions/min_length": 193.0,
|
|
"completions/max_length": 944.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 360.25,
|
|
"completions/min_terminated_length": 193.0,
|
|
"completions/max_terminated_length": 817.0,
|
|
"tools/call_frequency": 15.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.125,
|
|
"rewards/reward_func/std": 0.24848748743534088,
|
|
"reward": 0.125,
|
|
"reward_std": 0.24848748743534088,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001736155478283763,
|
|
"sampling/sampling_logp_difference/max": 0.5740090608596802,
|
|
"sampling/importance_sampling_ratio/min": 0.43841177225112915,
|
|
"sampling/importance_sampling_ratio/mean": 0.8835479021072388,
|
|
"sampling/importance_sampling_ratio/max": 1.453072190284729,
|
|
"kl": 0.0008838007834128803,
|
|
"entropy": 0.036205248325131834,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.342032000422478,
|
|
"epoch": 0.001015625,
|
|
"step": 39
|
|
},
|
|
{
|
|
"loss": 0.2130245417356491,
|
|
"grad_norm": 2.1143038272857666,
|
|
"learning_rate": 4.5e-07,
|
|
"num_tokens": 438814.0,
|
|
"completions/mean_length": 725.875,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 622.2000122070312,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 925.0,
|
|
"tools/call_frequency": 17.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18263888359069824,
|
|
"rewards/reward_func/std": 0.23096278309822083,
|
|
"reward": 0.18263888359069824,
|
|
"reward_std": 0.23096276819705963,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016259675612673163,
|
|
"sampling/sampling_logp_difference/max": 1.2221801280975342,
|
|
"sampling/importance_sampling_ratio/min": 0.14811836183071136,
|
|
"sampling/importance_sampling_ratio/mean": 0.6851245164871216,
|
|
"sampling/importance_sampling_ratio/max": 1.5476117134094238,
|
|
"kl": 0.0009807192368498363,
|
|
"entropy": 0.022754242818336934,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.509394301101565,
|
|
"epoch": 0.0010416666666666667,
|
|
"step": 40
|
|
},
|
|
{
|
|
"loss": 0.6120074987411499,
|
|
"grad_norm": 2.670624017715454,
|
|
"learning_rate": 4.482758620689655e-07,
|
|
"num_tokens": 448997.0,
|
|
"completions/mean_length": 588.375,
|
|
"completions/min_length": 177.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 536.7142944335938,
|
|
"completions/min_terminated_length": 177.0,
|
|
"completions/max_terminated_length": 915.0,
|
|
"tools/call_frequency": 14.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21562501788139343,
|
|
"rewards/reward_func/std": 0.33063453435897827,
|
|
"reward": 0.21562501788139343,
|
|
"reward_std": 0.33063453435897827,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0023741309996694326,
|
|
"sampling/sampling_logp_difference/max": 0.7091498374938965,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.113264799118042,
|
|
"sampling/importance_sampling_ratio/max": 2.8263494968414307,
|
|
"kl": 0.0011798903759654422,
|
|
"entropy": 0.054506233223946765,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.137434912845492,
|
|
"epoch": 0.0010677083333333333,
|
|
"step": 41
|
|
},
|
|
{
|
|
"loss": 0.0257548987865448,
|
|
"grad_norm": 1.425364375114441,
|
|
"learning_rate": 4.46551724137931e-07,
|
|
"num_tokens": 460484.0,
|
|
"completions/mean_length": 750.75,
|
|
"completions/min_length": 229.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 593.75,
|
|
"completions/min_terminated_length": 229.0,
|
|
"completions/max_terminated_length": 930.0,
|
|
"tools/call_frequency": 18.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.15312500298023224,
|
|
"rewards/reward_func/std": 0.23695899546146393,
|
|
"reward": 0.15312500298023224,
|
|
"reward_std": 0.23695896565914154,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001624097116291523,
|
|
"sampling/sampling_logp_difference/max": 0.5645525455474854,
|
|
"sampling/importance_sampling_ratio/min": 0.23968499898910522,
|
|
"sampling/importance_sampling_ratio/mean": 0.7106510996818542,
|
|
"sampling/importance_sampling_ratio/max": 1.2338413000106812,
|
|
"kl": 0.00149055604424575,
|
|
"entropy": 0.03127803542884067,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.9177359752357,
|
|
"epoch": 0.00109375,
|
|
"step": 42
|
|
},
|
|
{
|
|
"loss": 0.1516847312450409,
|
|
"grad_norm": 1.760916829109192,
|
|
"learning_rate": 4.4482758620689656e-07,
|
|
"num_tokens": 470818.0,
|
|
"completions/mean_length": 605.875,
|
|
"completions/min_length": 225.0,
|
|
"completions/max_length": 981.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 278.5,
|
|
"completions/min_terminated_length": 225.0,
|
|
"completions/max_terminated_length": 332.0,
|
|
"tools/call_frequency": 13.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.45885416865348816,
|
|
"rewards/reward_func/std": 0.30841729044914246,
|
|
"reward": 0.45885416865348816,
|
|
"reward_std": 0.30841726064682007,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0025465262588113546,
|
|
"sampling/sampling_logp_difference/max": 0.7246551513671875,
|
|
"sampling/importance_sampling_ratio/min": 0.13015159964561462,
|
|
"sampling/importance_sampling_ratio/mean": 0.9300558567047119,
|
|
"sampling/importance_sampling_ratio/max": 1.5529203414916992,
|
|
"kl": 0.001417617527749826,
|
|
"entropy": 0.052051851933356375,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.112876953557134,
|
|
"epoch": 0.0011197916666666667,
|
|
"step": 43
|
|
},
|
|
{
|
|
"loss": 0.01908973418176174,
|
|
"grad_norm": 2.998739719390869,
|
|
"learning_rate": 4.4310344827586207e-07,
|
|
"num_tokens": 480517.0,
|
|
"completions/mean_length": 526.25,
|
|
"completions/min_length": 241.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 408.3333435058594,
|
|
"completions/min_terminated_length": 241.0,
|
|
"completions/max_terminated_length": 890.0,
|
|
"tools/call_frequency": 12.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.28437498211860657,
|
|
"rewards/reward_func/std": 0.30264830589294434,
|
|
"reward": 0.28437498211860657,
|
|
"reward_std": 0.30264827609062195,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.004518352914601564,
|
|
"sampling/sampling_logp_difference/max": 0.6759898662567139,
|
|
"sampling/importance_sampling_ratio/min": 0.48342955112457275,
|
|
"sampling/importance_sampling_ratio/mean": 1.0765841007232666,
|
|
"sampling/importance_sampling_ratio/max": 1.8699613809585571,
|
|
"kl": 0.0013216507104516495,
|
|
"entropy": 0.06943492230493575,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.746606005355716,
|
|
"epoch": 0.0011458333333333333,
|
|
"step": 44
|
|
},
|
|
{
|
|
"loss": 0.3798207640647888,
|
|
"grad_norm": 1.485613226890564,
|
|
"learning_rate": 4.413793103448276e-07,
|
|
"num_tokens": 492563.0,
|
|
"completions/mean_length": 820.625,
|
|
"completions/min_length": 177.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 731.0,
|
|
"completions/min_terminated_length": 177.0,
|
|
"completions/max_terminated_length": 929.0,
|
|
"tools/call_frequency": 19.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.24523809552192688,
|
|
"rewards/reward_func/std": 0.3635886609554291,
|
|
"reward": 0.24523809552192688,
|
|
"reward_std": 0.3635886609554291,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014021442038938403,
|
|
"sampling/sampling_logp_difference/max": 0.7933025360107422,
|
|
"sampling/importance_sampling_ratio/min": 0.697462797164917,
|
|
"sampling/importance_sampling_ratio/mean": 1.1815321445465088,
|
|
"sampling/importance_sampling_ratio/max": 1.678512454032898,
|
|
"kl": 0.0008739797426642326,
|
|
"entropy": 0.0182966249412857,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.121435279026628,
|
|
"epoch": 0.001171875,
|
|
"step": 45
|
|
},
|
|
{
|
|
"loss": 0.09895484149456024,
|
|
"grad_norm": 3.2990074157714844,
|
|
"learning_rate": 4.396551724137931e-07,
|
|
"num_tokens": 503950.0,
|
|
"completions/mean_length": 739.75,
|
|
"completions/min_length": 173.0,
|
|
"completions/max_length": 959.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 562.5,
|
|
"completions/min_terminated_length": 173.0,
|
|
"completions/max_terminated_length": 922.0,
|
|
"tools/call_frequency": 17.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.11666667461395264,
|
|
"rewards/reward_func/std": 0.25463950634002686,
|
|
"reward": 0.11666667461395264,
|
|
"reward_std": 0.25463947653770447,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016792061505839229,
|
|
"sampling/sampling_logp_difference/max": 0.39375901222229004,
|
|
"sampling/importance_sampling_ratio/min": 0.4693974554538727,
|
|
"sampling/importance_sampling_ratio/mean": 1.007845163345337,
|
|
"sampling/importance_sampling_ratio/max": 1.680625557899475,
|
|
"kl": 0.0011395111714591621,
|
|
"entropy": 0.029940852778963745,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.02147406898439,
|
|
"epoch": 0.0011979166666666666,
|
|
"step": 46
|
|
},
|
|
{
|
|
"loss": 0.20527330040931702,
|
|
"grad_norm": 1.3684147596359253,
|
|
"learning_rate": 4.379310344827586e-07,
|
|
"num_tokens": 516047.0,
|
|
"completions/mean_length": 827.125,
|
|
"completions/min_length": 270.0,
|
|
"completions/max_length": 995.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 698.6666870117188,
|
|
"completions/min_terminated_length": 270.0,
|
|
"completions/max_terminated_length": 916.0,
|
|
"tools/call_frequency": 19.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17725694179534912,
|
|
"rewards/reward_func/std": 0.2007370889186859,
|
|
"reward": 0.17725694179534912,
|
|
"reward_std": 0.20073707401752472,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018504585605114698,
|
|
"sampling/sampling_logp_difference/max": 0.9017255306243896,
|
|
"sampling/importance_sampling_ratio/min": 0.2772412598133087,
|
|
"sampling/importance_sampling_ratio/mean": 0.9004523754119873,
|
|
"sampling/importance_sampling_ratio/max": 2.4420676231384277,
|
|
"kl": 0.0007191136101027951,
|
|
"entropy": 0.02769465235178359,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.866569634526968,
|
|
"epoch": 0.0012239583333333334,
|
|
"step": 47
|
|
},
|
|
{
|
|
"loss": 0.040666576474905014,
|
|
"grad_norm": 1.7948795557022095,
|
|
"learning_rate": 4.362068965517241e-07,
|
|
"num_tokens": 526775.0,
|
|
"completions/mean_length": 654.5,
|
|
"completions/min_length": 172.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 493.8000183105469,
|
|
"completions/min_terminated_length": 172.0,
|
|
"completions/max_terminated_length": 914.0,
|
|
"tools/call_frequency": 16.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.15206846594810486,
|
|
"rewards/reward_func/std": 0.28914350271224976,
|
|
"reward": 0.15206846594810486,
|
|
"reward_std": 0.28914353251457214,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015098638832569122,
|
|
"sampling/sampling_logp_difference/max": 0.45191335678100586,
|
|
"sampling/importance_sampling_ratio/min": 0.3150827884674072,
|
|
"sampling/importance_sampling_ratio/mean": 0.8162808418273926,
|
|
"sampling/importance_sampling_ratio/max": 1.0897504091262817,
|
|
"kl": 0.0005400325292228558,
|
|
"entropy": 0.03084199919248931,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.045441791415215,
|
|
"epoch": 0.00125,
|
|
"step": 48
|
|
},
|
|
{
|
|
"loss": -0.3227106034755707,
|
|
"grad_norm": 4.20354700088501,
|
|
"learning_rate": 4.344827586206896e-07,
|
|
"num_tokens": 538121.0,
|
|
"completions/mean_length": 733.5,
|
|
"completions/min_length": 172.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 423.3333435058594,
|
|
"completions/min_terminated_length": 172.0,
|
|
"completions/max_terminated_length": 920.0,
|
|
"tools/call_frequency": 16.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.012500000186264515,
|
|
"rewards/reward_func/std": 0.06943651288747787,
|
|
"reward": 0.012500000186264515,
|
|
"reward_std": 0.06943650543689728,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015484846662729979,
|
|
"sampling/sampling_logp_difference/max": 0.5872030258178711,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.0644381046295166,
|
|
"sampling/importance_sampling_ratio/max": 2.5321121215820312,
|
|
"kl": 0.0005724575407839438,
|
|
"entropy": 0.02865937410388142,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.949378801509738,
|
|
"epoch": 0.0012760416666666666,
|
|
"step": 49
|
|
},
|
|
{
|
|
"loss": -0.6737118363380432,
|
|
"grad_norm": 4.197883605957031,
|
|
"learning_rate": 4.327586206896551e-07,
|
|
"num_tokens": 546616.0,
|
|
"completions/mean_length": 376.375,
|
|
"completions/min_length": 178.0,
|
|
"completions/max_length": 881.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 308.2857360839844,
|
|
"completions/min_terminated_length": 178.0,
|
|
"completions/max_terminated_length": 881.0,
|
|
"tools/call_frequency": 9.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.08697916567325592,
|
|
"rewards/reward_func/std": 0.3044106960296631,
|
|
"reward": 0.08697916567325592,
|
|
"reward_std": 0.3044106662273407,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0032618786208331585,
|
|
"sampling/sampling_logp_difference/max": 0.4162408113479614,
|
|
"sampling/importance_sampling_ratio/min": 0.3647044897079468,
|
|
"sampling/importance_sampling_ratio/mean": 1.2134754657745361,
|
|
"sampling/importance_sampling_ratio/max": 2.682427406311035,
|
|
"kl": 0.0018156749792979099,
|
|
"entropy": 0.05371412669774145,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.408724637702107,
|
|
"epoch": 0.0013020833333333333,
|
|
"step": 50
|
|
},
|
|
{
|
|
"loss": -0.05693481117486954,
|
|
"grad_norm": 3.372880697250366,
|
|
"learning_rate": 4.310344827586206e-07,
|
|
"num_tokens": 558582.0,
|
|
"completions/mean_length": 811.125,
|
|
"completions/min_length": 176.0,
|
|
"completions/max_length": 942.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 740.0,
|
|
"completions/min_terminated_length": 176.0,
|
|
"completions/max_terminated_length": 942.0,
|
|
"tools/call_frequency": 19.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13437499105930328,
|
|
"rewards/reward_func/std": 0.22992569208145142,
|
|
"reward": 0.13437499105930328,
|
|
"reward_std": 0.22992569208145142,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0011997855035588145,
|
|
"sampling/sampling_logp_difference/max": 0.8754709959030151,
|
|
"sampling/importance_sampling_ratio/min": 0.4884870648384094,
|
|
"sampling/importance_sampling_ratio/mean": 1.3136510848999023,
|
|
"sampling/importance_sampling_ratio/max": 2.8247694969177246,
|
|
"kl": 0.0004776921753091301,
|
|
"entropy": 0.01786074467236176,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.861001746729016,
|
|
"epoch": 0.001328125,
|
|
"step": 51
|
|
},
|
|
{
|
|
"loss": -0.4004557132720947,
|
|
"grad_norm": 2.8344621658325195,
|
|
"learning_rate": 4.2931034482758624e-07,
|
|
"num_tokens": 569888.0,
|
|
"completions/mean_length": 726.875,
|
|
"completions/min_length": 245.0,
|
|
"completions/max_length": 911.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 630.2000122070312,
|
|
"completions/min_terminated_length": 245.0,
|
|
"completions/max_terminated_length": 884.0,
|
|
"tools/call_frequency": 17.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.24166667461395264,
|
|
"rewards/reward_func/std": 0.2507704198360443,
|
|
"reward": 0.24166667461395264,
|
|
"reward_std": 0.2507704198360443,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001461285282857716,
|
|
"sampling/sampling_logp_difference/max": 0.4744536280632019,
|
|
"sampling/importance_sampling_ratio/min": 0.35955381393432617,
|
|
"sampling/importance_sampling_ratio/mean": 1.2819757461547852,
|
|
"sampling/importance_sampling_ratio/max": 2.664541721343994,
|
|
"kl": 0.0007812018937158882,
|
|
"entropy": 0.02665649395203218,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.990700993686914,
|
|
"epoch": 0.0013541666666666667,
|
|
"step": 52
|
|
},
|
|
{
|
|
"loss": -0.1753660887479782,
|
|
"grad_norm": 3.575989007949829,
|
|
"learning_rate": 4.2758620689655174e-07,
|
|
"num_tokens": 580434.0,
|
|
"completions/mean_length": 633.375,
|
|
"completions/min_length": 142.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 381.75,
|
|
"completions/min_terminated_length": 142.0,
|
|
"completions/max_terminated_length": 913.0,
|
|
"tools/call_frequency": 15.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2083333432674408,
|
|
"rewards/reward_func/std": 0.28203484416007996,
|
|
"reward": 0.2083333432674408,
|
|
"reward_std": 0.28203484416007996,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017873237375169992,
|
|
"sampling/sampling_logp_difference/max": 1.12148118019104,
|
|
"sampling/importance_sampling_ratio/min": 0.12836112082004547,
|
|
"sampling/importance_sampling_ratio/mean": 0.9751670360565186,
|
|
"sampling/importance_sampling_ratio/max": 1.7498493194580078,
|
|
"kl": 0.0003591144268284552,
|
|
"entropy": 0.025020689063239843,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.215505322441459,
|
|
"epoch": 0.0013802083333333333,
|
|
"step": 53
|
|
},
|
|
{
|
|
"loss": -0.028369426727294922,
|
|
"grad_norm": 3.6068708896636963,
|
|
"learning_rate": 4.2586206896551725e-07,
|
|
"num_tokens": 589023.0,
|
|
"completions/mean_length": 389.125,
|
|
"completions/min_length": 149.0,
|
|
"completions/max_length": 900.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 219.83334350585938,
|
|
"completions/min_terminated_length": 149.0,
|
|
"completions/max_terminated_length": 250.0,
|
|
"tools/call_frequency": 9.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.09687499701976776,
|
|
"rewards/reward_func/std": 0.3279910385608673,
|
|
"reward": 0.09687499701976776,
|
|
"reward_std": 0.3279910385608673,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0041470336727797985,
|
|
"sampling/sampling_logp_difference/max": 0.4696468114852905,
|
|
"sampling/importance_sampling_ratio/min": 0.5189381241798401,
|
|
"sampling/importance_sampling_ratio/mean": 1.0449053049087524,
|
|
"sampling/importance_sampling_ratio/max": 2.5507614612579346,
|
|
"kl": 0.0013719275484618265,
|
|
"entropy": 0.0587151714717038,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.214799202978611,
|
|
"epoch": 0.00140625,
|
|
"step": 54
|
|
},
|
|
{
|
|
"loss": -0.552887499332428,
|
|
"grad_norm": 5.890571594238281,
|
|
"learning_rate": 4.2413793103448276e-07,
|
|
"num_tokens": 596906.0,
|
|
"completions/mean_length": 300.625,
|
|
"completions/min_length": 79.0,
|
|
"completions/max_length": 822.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 226.1428680419922,
|
|
"completions/min_terminated_length": 79.0,
|
|
"completions/max_terminated_length": 388.0,
|
|
"tools/call_frequency": 7.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1302083283662796,
|
|
"rewards/reward_func/std": 0.38750162720680237,
|
|
"reward": 0.1302083283662796,
|
|
"reward_std": 0.38750159740448,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.004885816015303135,
|
|
"sampling/sampling_logp_difference/max": 1.247067928314209,
|
|
"sampling/importance_sampling_ratio/min": 0.2358967810869217,
|
|
"sampling/importance_sampling_ratio/mean": 0.699617862701416,
|
|
"sampling/importance_sampling_ratio/max": 1.4744852781295776,
|
|
"kl": 0.002767269406831474,
|
|
"entropy": 0.07449608249589801,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.328409168869257,
|
|
"epoch": 0.0014322916666666666,
|
|
"step": 55
|
|
},
|
|
{
|
|
"loss": -0.3186432123184204,
|
|
"grad_norm": 2.6123993396759033,
|
|
"learning_rate": 4.2241379310344827e-07,
|
|
"num_tokens": 607549.0,
|
|
"completions/mean_length": 644.875,
|
|
"completions/min_length": 178.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 375.75,
|
|
"completions/min_terminated_length": 178.0,
|
|
"completions/max_terminated_length": 905.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": -0.0062500000931322575,
|
|
"rewards/reward_func/std": 0.07763238251209259,
|
|
"reward": -0.0062500000931322575,
|
|
"reward_std": 0.077632375061512,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015098920557647943,
|
|
"sampling/sampling_logp_difference/max": 0.6079216003417969,
|
|
"sampling/importance_sampling_ratio/min": 0.23547157645225525,
|
|
"sampling/importance_sampling_ratio/mean": 0.9280757308006287,
|
|
"sampling/importance_sampling_ratio/max": 2.069110870361328,
|
|
"kl": 0.001868254159035132,
|
|
"entropy": 0.03243341337656602,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.201451057568192,
|
|
"epoch": 0.0014583333333333334,
|
|
"step": 56
|
|
},
|
|
{
|
|
"loss": 0.04303571954369545,
|
|
"grad_norm": 3.3504638671875,
|
|
"learning_rate": 4.206896551724138e-07,
|
|
"num_tokens": 617717.0,
|
|
"completions/mean_length": 586.375,
|
|
"completions/min_length": 195.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 254.5,
|
|
"completions/min_terminated_length": 195.0,
|
|
"completions/max_terminated_length": 303.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1197916641831398,
|
|
"rewards/reward_func/std": 0.27523213624954224,
|
|
"reward": 0.1197916641831398,
|
|
"reward_std": 0.27523213624954224,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020454139448702335,
|
|
"sampling/sampling_logp_difference/max": 0.4852004051208496,
|
|
"sampling/importance_sampling_ratio/min": 0.8056414723396301,
|
|
"sampling/importance_sampling_ratio/mean": 1.1623942852020264,
|
|
"sampling/importance_sampling_ratio/max": 1.738699197769165,
|
|
"kl": 0.0007646686062230401,
|
|
"entropy": 0.04275305586634204,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.529754409566522,
|
|
"epoch": 0.001484375,
|
|
"step": 57
|
|
},
|
|
{
|
|
"loss": 0.00028277630917727947,
|
|
"grad_norm": 0.606818437576294,
|
|
"learning_rate": 4.189655172413793e-07,
|
|
"num_tokens": 630522.0,
|
|
"completions/mean_length": 914.75,
|
|
"completions/min_length": 846.0,
|
|
"completions/max_length": 951.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 885.0,
|
|
"completions/min_terminated_length": 846.0,
|
|
"completions/max_terminated_length": 921.0,
|
|
"tools/call_frequency": 22.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05000000074505806,
|
|
"rewards/reward_func/std": 0.0,
|
|
"reward": 0.05000000074505806,
|
|
"reward_std": 0.0,
|
|
"frac_reward_zero_std": 1.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013281708816066384,
|
|
"sampling/sampling_logp_difference/max": 0.698073148727417,
|
|
"sampling/importance_sampling_ratio/min": 0.5691529512405396,
|
|
"sampling/importance_sampling_ratio/mean": 1.1723623275756836,
|
|
"sampling/importance_sampling_ratio/max": 2.569185495376587,
|
|
"kl": 0.00029148480132334953,
|
|
"entropy": 0.015984372090315446,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.443296758458018,
|
|
"epoch": 0.0015104166666666666,
|
|
"step": 58
|
|
},
|
|
{
|
|
"loss": 0.06852003931999207,
|
|
"grad_norm": 5.16364049911499,
|
|
"learning_rate": 4.172413793103448e-07,
|
|
"num_tokens": 640701.0,
|
|
"completions/mean_length": 586.625,
|
|
"completions/min_length": 217.0,
|
|
"completions/max_length": 963.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 480.0,
|
|
"completions/min_terminated_length": 217.0,
|
|
"completions/max_terminated_length": 963.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10295139253139496,
|
|
"rewards/reward_func/std": 0.20155704021453857,
|
|
"reward": 0.10295139253139496,
|
|
"reward_std": 0.20155705511569977,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0029931282624602318,
|
|
"sampling/sampling_logp_difference/max": 0.7177779674530029,
|
|
"sampling/importance_sampling_ratio/min": 0.48087671399116516,
|
|
"sampling/importance_sampling_ratio/mean": 1.2119784355163574,
|
|
"sampling/importance_sampling_ratio/max": 1.87960684299469,
|
|
"kl": 0.0013817033213854302,
|
|
"entropy": 0.0624709160765633,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.571871334686875,
|
|
"epoch": 0.0015364583333333333,
|
|
"step": 59
|
|
},
|
|
{
|
|
"loss": 0.09876267611980438,
|
|
"grad_norm": 2.148210287094116,
|
|
"learning_rate": 4.155172413793103e-07,
|
|
"num_tokens": 651372.0,
|
|
"completions/mean_length": 648.25,
|
|
"completions/min_length": 174.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 389.75,
|
|
"completions/min_terminated_length": 174.0,
|
|
"completions/max_terminated_length": 929.0,
|
|
"tools/call_frequency": 15.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10104166716337204,
|
|
"rewards/reward_func/std": 0.2740943729877472,
|
|
"reward": 0.10104166716337204,
|
|
"reward_std": 0.2740943729877472,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020331619307398796,
|
|
"sampling/sampling_logp_difference/max": 0.7180962562561035,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8786753416061401,
|
|
"sampling/importance_sampling_ratio/max": 2.150700569152832,
|
|
"kl": 0.0007783782513115511,
|
|
"entropy": 0.029667860304471105,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.418167762458324,
|
|
"epoch": 0.0015625,
|
|
"step": 60
|
|
},
|
|
{
|
|
"loss": 0.18817365169525146,
|
|
"grad_norm": 1.0397121906280518,
|
|
"learning_rate": 4.1379310344827586e-07,
|
|
"num_tokens": 662722.0,
|
|
"completions/mean_length": 733.75,
|
|
"completions/min_length": 229.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 567.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"completions/max_terminated_length": 901.0,
|
|
"tools/call_frequency": 18.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.14374999701976776,
|
|
"rewards/reward_func/std": 0.3299756646156311,
|
|
"reward": 0.14374999701976776,
|
|
"reward_std": 0.3299756348133087,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014046495780348778,
|
|
"sampling/sampling_logp_difference/max": 0.44372260570526123,
|
|
"sampling/importance_sampling_ratio/min": 0.3140422999858856,
|
|
"sampling/importance_sampling_ratio/mean": 0.7978009581565857,
|
|
"sampling/importance_sampling_ratio/max": 1.7694478034973145,
|
|
"kl": 0.0004985993809896172,
|
|
"entropy": 0.02953001018613577,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.198667164891958,
|
|
"epoch": 0.0015885416666666667,
|
|
"step": 61
|
|
},
|
|
{
|
|
"loss": 0.00016799441073089838,
|
|
"grad_norm": 0.2792870104312897,
|
|
"learning_rate": 4.1206896551724137e-07,
|
|
"num_tokens": 675576.0,
|
|
"completions/mean_length": 922.625,
|
|
"completions/min_length": 898.0,
|
|
"completions/max_length": 976.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 920.5,
|
|
"completions/min_terminated_length": 915.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 21.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05000000074505806,
|
|
"rewards/reward_func/std": 0.0,
|
|
"reward": 0.05000000074505806,
|
|
"reward_std": 0.0,
|
|
"frac_reward_zero_std": 1.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0011363234370946884,
|
|
"sampling/sampling_logp_difference/max": 0.4669325351715088,
|
|
"sampling/importance_sampling_ratio/min": 0.5352997183799744,
|
|
"sampling/importance_sampling_ratio/mean": 0.9600976705551147,
|
|
"sampling/importance_sampling_ratio/max": 1.9824895858764648,
|
|
"kl": 0.00016763195208113757,
|
|
"entropy": 0.012928913929499686,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.657696371898055,
|
|
"epoch": 0.0016145833333333333,
|
|
"step": 62
|
|
},
|
|
{
|
|
"loss": 0.031228993088006973,
|
|
"grad_norm": 1.546233892440796,
|
|
"learning_rate": 4.103448275862069e-07,
|
|
"num_tokens": 687791.0,
|
|
"completions/mean_length": 841.75,
|
|
"completions/min_length": 220.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 751.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"completions/max_terminated_length": 952.0,
|
|
"tools/call_frequency": 18.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.14374999701976776,
|
|
"rewards/reward_func/std": 0.18163442611694336,
|
|
"reward": 0.14374999701976776,
|
|
"reward_std": 0.18163444101810455,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0012039071880280972,
|
|
"sampling/sampling_logp_difference/max": 0.8301470279693604,
|
|
"sampling/importance_sampling_ratio/min": 0.2306743711233139,
|
|
"sampling/importance_sampling_ratio/mean": 0.8433521389961243,
|
|
"sampling/importance_sampling_ratio/max": 1.2378512620925903,
|
|
"kl": 0.00026068457893302366,
|
|
"entropy": 0.018836127448594198,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.801032239571214,
|
|
"epoch": 0.001640625,
|
|
"step": 63
|
|
},
|
|
{
|
|
"loss": -0.03366469219326973,
|
|
"grad_norm": 3.432100296020508,
|
|
"learning_rate": 4.086206896551724e-07,
|
|
"num_tokens": 698573.0,
|
|
"completions/mean_length": 662.625,
|
|
"completions/min_length": 175.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 241.6666717529297,
|
|
"completions/min_terminated_length": 175.0,
|
|
"completions/max_terminated_length": 318.0,
|
|
"tools/call_frequency": 15.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.19427083432674408,
|
|
"rewards/reward_func/std": 0.27452102303504944,
|
|
"reward": 0.19427083432674408,
|
|
"reward_std": 0.27452102303504944,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.00202364195138216,
|
|
"sampling/sampling_logp_difference/max": 0.6172208786010742,
|
|
"sampling/importance_sampling_ratio/min": 0.4787842035293579,
|
|
"sampling/importance_sampling_ratio/mean": 1.0496106147766113,
|
|
"sampling/importance_sampling_ratio/max": 2.8589730262756348,
|
|
"kl": 0.0009462613907658124,
|
|
"entropy": 0.03347723593469709,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.696901548653841,
|
|
"epoch": 0.0016666666666666668,
|
|
"step": 64
|
|
},
|
|
{
|
|
"loss": -0.26673662662506104,
|
|
"grad_norm": 5.773024559020996,
|
|
"learning_rate": 4.068965517241379e-07,
|
|
"num_tokens": 709301.0,
|
|
"completions/mean_length": 656.5,
|
|
"completions/min_length": 182.0,
|
|
"completions/max_length": 949.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 400.5,
|
|
"completions/min_terminated_length": 182.0,
|
|
"completions/max_terminated_length": 886.0,
|
|
"tools/call_frequency": 15.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2744034230709076,
|
|
"rewards/reward_func/std": 0.3807433843612671,
|
|
"reward": 0.2744034230709076,
|
|
"reward_std": 0.3807433843612671,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019280033884570003,
|
|
"sampling/sampling_logp_difference/max": 0.7840023040771484,
|
|
"sampling/importance_sampling_ratio/min": 0.49734193086624146,
|
|
"sampling/importance_sampling_ratio/mean": 1.2997841835021973,
|
|
"sampling/importance_sampling_ratio/max": 2.4130196571350098,
|
|
"kl": 0.0010537478237893083,
|
|
"entropy": 0.04225548525573686,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.697153873741627,
|
|
"epoch": 0.0016927083333333334,
|
|
"step": 65
|
|
},
|
|
{
|
|
"loss": 0.31974104046821594,
|
|
"grad_norm": 3.941054105758667,
|
|
"learning_rate": 4.051724137931034e-07,
|
|
"num_tokens": 718546.0,
|
|
"completions/mean_length": 471.0,
|
|
"completions/min_length": 154.0,
|
|
"completions/max_length": 917.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 411.2857360839844,
|
|
"completions/min_terminated_length": 154.0,
|
|
"completions/max_terminated_length": 917.0,
|
|
"tools/call_frequency": 11.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2604166567325592,
|
|
"rewards/reward_func/std": 0.379137247800827,
|
|
"reward": 0.2604166567325592,
|
|
"reward_std": 0.379137247800827,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0035213667433708906,
|
|
"sampling/sampling_logp_difference/max": 0.564582109451294,
|
|
"sampling/importance_sampling_ratio/min": 0.1714719831943512,
|
|
"sampling/importance_sampling_ratio/mean": 0.9359852075576782,
|
|
"sampling/importance_sampling_ratio/max": 1.8976486921310425,
|
|
"kl": 0.002775293679405877,
|
|
"entropy": 0.05870910652447492,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.26980372145772,
|
|
"epoch": 0.00171875,
|
|
"step": 66
|
|
},
|
|
{
|
|
"loss": -0.33820250630378723,
|
|
"grad_norm": 2.8793344497680664,
|
|
"learning_rate": 4.034482758620689e-07,
|
|
"num_tokens": 729238.0,
|
|
"completions/mean_length": 651.875,
|
|
"completions/min_length": 172.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 372.5,
|
|
"completions/min_terminated_length": 172.0,
|
|
"completions/max_terminated_length": 873.0,
|
|
"tools/call_frequency": 15.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": -0.0062500000931322575,
|
|
"rewards/reward_func/std": 0.07763238251209259,
|
|
"reward": -0.0062500000931322575,
|
|
"reward_std": 0.077632375061512,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013644435675814748,
|
|
"sampling/sampling_logp_difference/max": 0.4909186363220215,
|
|
"sampling/importance_sampling_ratio/min": 0.6196874976158142,
|
|
"sampling/importance_sampling_ratio/mean": 1.2284507751464844,
|
|
"sampling/importance_sampling_ratio/max": 2.5221643447875977,
|
|
"kl": 0.0015754469372950552,
|
|
"entropy": 0.02263550434145145,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.782281374558806,
|
|
"epoch": 0.0017447916666666666,
|
|
"step": 67
|
|
},
|
|
{
|
|
"loss": -0.21227873861789703,
|
|
"grad_norm": 2.758469343185425,
|
|
"learning_rate": 4.017241379310345e-07,
|
|
"num_tokens": 739935.0,
|
|
"completions/mean_length": 650.375,
|
|
"completions/min_length": 159.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 505.0,
|
|
"completions/min_terminated_length": 159.0,
|
|
"completions/max_terminated_length": 928.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06145833805203438,
|
|
"rewards/reward_func/std": 0.13793671131134033,
|
|
"reward": 0.06145833805203438,
|
|
"reward_std": 0.13793671131134033,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002464109333232045,
|
|
"sampling/sampling_logp_difference/max": 0.564541220664978,
|
|
"sampling/importance_sampling_ratio/min": 0.456142395734787,
|
|
"sampling/importance_sampling_ratio/mean": 1.1993980407714844,
|
|
"sampling/importance_sampling_ratio/max": 2.0841262340545654,
|
|
"kl": 0.0008561946779082064,
|
|
"entropy": 0.042545791482552886,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.201567942276597,
|
|
"epoch": 0.0017708333333333332,
|
|
"step": 68
|
|
},
|
|
{
|
|
"loss": 0.2750667929649353,
|
|
"grad_norm": 2.0811007022857666,
|
|
"learning_rate": 4e-07,
|
|
"num_tokens": 750851.0,
|
|
"completions/mean_length": 680.25,
|
|
"completions/min_length": 207.0,
|
|
"completions/max_length": 1002.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 523.0,
|
|
"completions/min_terminated_length": 207.0,
|
|
"completions/max_terminated_length": 942.0,
|
|
"tools/call_frequency": 14.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.22361111640930176,
|
|
"rewards/reward_func/std": 0.2743840217590332,
|
|
"reward": 0.22361111640930176,
|
|
"reward_std": 0.2743840217590332,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019310215720906854,
|
|
"sampling/sampling_logp_difference/max": 1.6195259094238281,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.9248746633529663,
|
|
"sampling/importance_sampling_ratio/max": 1.3823113441467285,
|
|
"kl": 0.0016639361974739586,
|
|
"entropy": 0.04270535719115287,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.599187646061182,
|
|
"epoch": 0.001796875,
|
|
"step": 69
|
|
},
|
|
{
|
|
"loss": 0.15652912855148315,
|
|
"grad_norm": 2.376732349395752,
|
|
"learning_rate": 3.9827586206896554e-07,
|
|
"num_tokens": 762051.0,
|
|
"completions/mean_length": 714.125,
|
|
"completions/min_length": 71.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 590.4000244140625,
|
|
"completions/min_terminated_length": 71.0,
|
|
"completions/max_terminated_length": 939.0,
|
|
"tools/call_frequency": 17.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.14687499403953552,
|
|
"rewards/reward_func/std": 0.3387048542499542,
|
|
"reward": 0.14687499403953552,
|
|
"reward_std": 0.3387048840522766,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0010694247903302312,
|
|
"sampling/sampling_logp_difference/max": 0.5529055595397949,
|
|
"sampling/importance_sampling_ratio/min": 0.4688456654548645,
|
|
"sampling/importance_sampling_ratio/mean": 0.9322454929351807,
|
|
"sampling/importance_sampling_ratio/max": 1.6051846742630005,
|
|
"kl": 0.0012747324417432537,
|
|
"entropy": 0.057399596786126494,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.527622018009424,
|
|
"epoch": 0.0018229166666666667,
|
|
"step": 70
|
|
},
|
|
{
|
|
"loss": 0.34723353385925293,
|
|
"grad_norm": 1.928682804107666,
|
|
"learning_rate": 3.9655172413793105e-07,
|
|
"num_tokens": 772798.0,
|
|
"completions/mean_length": 658.125,
|
|
"completions/min_length": 224.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 499.6000061035156,
|
|
"completions/min_terminated_length": 224.0,
|
|
"completions/max_terminated_length": 918.0,
|
|
"tools/call_frequency": 15.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21979166567325592,
|
|
"rewards/reward_func/std": 0.36634019017219543,
|
|
"reward": 0.21979166567325592,
|
|
"reward_std": 0.36634016036987305,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020385142415761948,
|
|
"sampling/sampling_logp_difference/max": 0.6180412769317627,
|
|
"sampling/importance_sampling_ratio/min": 0.3175122141838074,
|
|
"sampling/importance_sampling_ratio/mean": 1.054479956626892,
|
|
"sampling/importance_sampling_ratio/max": 1.7177965641021729,
|
|
"kl": 0.001775132934199064,
|
|
"entropy": 0.03358624747488648,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.359526578336954,
|
|
"epoch": 0.0018489583333333333,
|
|
"step": 71
|
|
},
|
|
{
|
|
"loss": 0.1096133142709732,
|
|
"grad_norm": 2.851795196533203,
|
|
"learning_rate": 3.9482758620689656e-07,
|
|
"num_tokens": 783619.0,
|
|
"completions/mean_length": 666.75,
|
|
"completions/min_length": 180.0,
|
|
"completions/max_length": 962.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 514.2000122070312,
|
|
"completions/min_terminated_length": 180.0,
|
|
"completions/max_terminated_length": 962.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21458333730697632,
|
|
"rewards/reward_func/std": 0.3018050789833069,
|
|
"reward": 0.21458333730697632,
|
|
"reward_std": 0.3018050789833069,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016206916188821197,
|
|
"sampling/sampling_logp_difference/max": 0.47581905126571655,
|
|
"sampling/importance_sampling_ratio/min": 0.6126041412353516,
|
|
"sampling/importance_sampling_ratio/mean": 0.9244512915611267,
|
|
"sampling/importance_sampling_ratio/max": 1.1447086334228516,
|
|
"kl": 0.0011197467993042665,
|
|
"entropy": 0.03440619522007182,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.437525160610676,
|
|
"epoch": 0.001875,
|
|
"step": 72
|
|
},
|
|
{
|
|
"loss": 0.25123533606529236,
|
|
"grad_norm": 2.5729682445526123,
|
|
"learning_rate": 3.9310344827586207e-07,
|
|
"num_tokens": 795103.0,
|
|
"completions/mean_length": 750.375,
|
|
"completions/min_length": 206.0,
|
|
"completions/max_length": 951.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 576.75,
|
|
"completions/min_terminated_length": 206.0,
|
|
"completions/max_terminated_length": 940.0,
|
|
"tools/call_frequency": 17.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2829861044883728,
|
|
"rewards/reward_func/std": 0.3285786807537079,
|
|
"reward": 0.2829861044883728,
|
|
"reward_std": 0.3285786509513855,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015425707679241896,
|
|
"sampling/sampling_logp_difference/max": 0.4835818409919739,
|
|
"sampling/importance_sampling_ratio/min": 0.5805332064628601,
|
|
"sampling/importance_sampling_ratio/mean": 1.035643458366394,
|
|
"sampling/importance_sampling_ratio/max": 2.329007863998413,
|
|
"kl": 0.0010214885556365516,
|
|
"entropy": 0.03248234454076737,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.584036394953728,
|
|
"epoch": 0.0019010416666666668,
|
|
"step": 73
|
|
},
|
|
{
|
|
"loss": 0.0686991736292839,
|
|
"grad_norm": 1.7932507991790771,
|
|
"learning_rate": 3.913793103448276e-07,
|
|
"num_tokens": 807221.0,
|
|
"completions/mean_length": 829.875,
|
|
"completions/min_length": 270.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 560.5,
|
|
"completions/min_terminated_length": 270.0,
|
|
"completions/max_terminated_length": 851.0,
|
|
"tools/call_frequency": 18.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3375000059604645,
|
|
"rewards/reward_func/std": 0.3178783357143402,
|
|
"reward": 0.3375000059604645,
|
|
"reward_std": 0.3178783357143402,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001482333056628704,
|
|
"sampling/sampling_logp_difference/max": 0.6158363819122314,
|
|
"sampling/importance_sampling_ratio/min": 0.4821794033050537,
|
|
"sampling/importance_sampling_ratio/mean": 0.8979474902153015,
|
|
"sampling/importance_sampling_ratio/max": 2.1596951484680176,
|
|
"kl": 0.0009881261150894716,
|
|
"entropy": 0.02165689109824598,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.890293288975954,
|
|
"epoch": 0.0019270833333333334,
|
|
"step": 74
|
|
},
|
|
{
|
|
"loss": 1.022073745727539,
|
|
"grad_norm": 3.2335641384124756,
|
|
"learning_rate": 3.896551724137931e-07,
|
|
"num_tokens": 816673.0,
|
|
"completions/mean_length": 496.875,
|
|
"completions/min_length": 227.0,
|
|
"completions/max_length": 940.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 439.4285888671875,
|
|
"completions/min_terminated_length": 227.0,
|
|
"completions/max_terminated_length": 940.0,
|
|
"tools/call_frequency": 12.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.5121527910232544,
|
|
"rewards/reward_func/std": 0.31924229860305786,
|
|
"reward": 0.5121527910232544,
|
|
"reward_std": 0.31924229860305786,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0032130698673427105,
|
|
"sampling/sampling_logp_difference/max": 1.265617847442627,
|
|
"sampling/importance_sampling_ratio/min": 0.3573102056980133,
|
|
"sampling/importance_sampling_ratio/mean": 1.1924068927764893,
|
|
"sampling/importance_sampling_ratio/max": 2.2950894832611084,
|
|
"kl": 0.0024906754406401888,
|
|
"entropy": 0.05731054220814258,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.14017285220325,
|
|
"epoch": 0.001953125,
|
|
"step": 75
|
|
},
|
|
{
|
|
"loss": 0.1207452192902565,
|
|
"grad_norm": 1.818686842918396,
|
|
"learning_rate": 3.879310344827586e-07,
|
|
"num_tokens": 826891.0,
|
|
"completions/mean_length": 592.5,
|
|
"completions/min_length": 209.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 265.5,
|
|
"completions/min_terminated_length": 209.0,
|
|
"completions/max_terminated_length": 361.0,
|
|
"tools/call_frequency": 13.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.171875,
|
|
"rewards/reward_func/std": 0.3441961407661438,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.3441961407661438,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0026178148109465837,
|
|
"sampling/sampling_logp_difference/max": 0.6253746747970581,
|
|
"sampling/importance_sampling_ratio/min": 0.5166544914245605,
|
|
"sampling/importance_sampling_ratio/mean": 1.079270601272583,
|
|
"sampling/importance_sampling_ratio/max": 2.324326515197754,
|
|
"kl": 0.0010160994506804855,
|
|
"entropy": 0.05011388775892556,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.038167165592313,
|
|
"epoch": 0.001979166666666667,
|
|
"step": 76
|
|
},
|
|
{
|
|
"loss": 0.12573407590389252,
|
|
"grad_norm": 2.030879259109497,
|
|
"learning_rate": 3.862068965517241e-07,
|
|
"num_tokens": 838425.0,
|
|
"completions/mean_length": 755.625,
|
|
"completions/min_length": 237.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 572.5,
|
|
"completions/min_terminated_length": 237.0,
|
|
"completions/max_terminated_length": 901.0,
|
|
"tools/call_frequency": 17.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.12725694477558136,
|
|
"rewards/reward_func/std": 0.21217353641986847,
|
|
"reward": 0.12725694477558136,
|
|
"reward_std": 0.21217352151870728,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013723166193813086,
|
|
"sampling/sampling_logp_difference/max": 0.627720832824707,
|
|
"sampling/importance_sampling_ratio/min": 0.4538462460041046,
|
|
"sampling/importance_sampling_ratio/mean": 1.0086034536361694,
|
|
"sampling/importance_sampling_ratio/max": 1.9921493530273438,
|
|
"kl": 0.0009068485187526676,
|
|
"entropy": 0.02431450749281794,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.692305989563465,
|
|
"epoch": 0.0020052083333333332,
|
|
"step": 77
|
|
},
|
|
{
|
|
"loss": 0.46705156564712524,
|
|
"grad_norm": 3.1631641387939453,
|
|
"learning_rate": 3.844827586206896e-07,
|
|
"num_tokens": 849234.0,
|
|
"completions/mean_length": 666.0,
|
|
"completions/min_length": 174.0,
|
|
"completions/max_length": 949.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 235.33334350585938,
|
|
"completions/min_terminated_length": 174.0,
|
|
"completions/max_terminated_length": 285.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2696875035762787,
|
|
"rewards/reward_func/std": 0.3228124976158142,
|
|
"reward": 0.2696875035762787,
|
|
"reward_std": 0.3228125274181366,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022310747299343348,
|
|
"sampling/sampling_logp_difference/max": 0.9685945510864258,
|
|
"sampling/importance_sampling_ratio/min": 0.5834606885910034,
|
|
"sampling/importance_sampling_ratio/mean": 1.0331593751907349,
|
|
"sampling/importance_sampling_ratio/max": 1.8925235271453857,
|
|
"kl": 0.0029393876953185405,
|
|
"entropy": 0.0425157661375124,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.219812221825123,
|
|
"epoch": 0.00203125,
|
|
"step": 78
|
|
},
|
|
{
|
|
"loss": 0.04879586771130562,
|
|
"grad_norm": 2.6453773975372314,
|
|
"learning_rate": 3.827586206896551e-07,
|
|
"num_tokens": 860698.0,
|
|
"completions/mean_length": 748.875,
|
|
"completions/min_length": 253.0,
|
|
"completions/max_length": 935.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 476.66668701171875,
|
|
"completions/min_terminated_length": 253.0,
|
|
"completions/max_terminated_length": 884.0,
|
|
"tools/call_frequency": 17.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3531250059604645,
|
|
"rewards/reward_func/std": 0.4184233844280243,
|
|
"reward": 0.3531250059604645,
|
|
"reward_std": 0.4184233844280243,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014507040614262223,
|
|
"sampling/sampling_logp_difference/max": 0.42525625228881836,
|
|
"sampling/importance_sampling_ratio/min": 0.5543180704116821,
|
|
"sampling/importance_sampling_ratio/mean": 1.0547351837158203,
|
|
"sampling/importance_sampling_ratio/max": 2.648834705352783,
|
|
"kl": 0.000787372995887381,
|
|
"entropy": 0.03333819209365174,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.28350891545415,
|
|
"epoch": 0.0020572916666666665,
|
|
"step": 79
|
|
},
|
|
{
|
|
"loss": 0.1860741525888443,
|
|
"grad_norm": 1.6004518270492554,
|
|
"learning_rate": 3.810344827586207e-07,
|
|
"num_tokens": 871304.0,
|
|
"completions/mean_length": 640.625,
|
|
"completions/min_length": 144.0,
|
|
"completions/max_length": 933.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 469.6000061035156,
|
|
"completions/min_terminated_length": 144.0,
|
|
"completions/max_terminated_length": 932.0,
|
|
"tools/call_frequency": 14.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2770833373069763,
|
|
"rewards/reward_func/std": 0.38138091564178467,
|
|
"reward": 0.2770833373069763,
|
|
"reward_std": 0.38138091564178467,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001821009675040841,
|
|
"sampling/sampling_logp_difference/max": 0.6232120990753174,
|
|
"sampling/importance_sampling_ratio/min": 0.40542325377464294,
|
|
"sampling/importance_sampling_ratio/mean": 0.8059606552124023,
|
|
"sampling/importance_sampling_ratio/max": 1.4009113311767578,
|
|
"kl": 0.0015353300004790071,
|
|
"entropy": 0.03338964283466339,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.055674178525805,
|
|
"epoch": 0.0020833333333333333,
|
|
"step": 80
|
|
},
|
|
{
|
|
"loss": 0.252197265625,
|
|
"grad_norm": 4.594541549682617,
|
|
"learning_rate": 3.793103448275862e-07,
|
|
"num_tokens": 880629.0,
|
|
"completions/mean_length": 479.375,
|
|
"completions/min_length": 101.0,
|
|
"completions/max_length": 927.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 418.14288330078125,
|
|
"completions/min_terminated_length": 101.0,
|
|
"completions/max_terminated_length": 927.0,
|
|
"tools/call_frequency": 12.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23593750596046448,
|
|
"rewards/reward_func/std": 0.3606441020965576,
|
|
"reward": 0.23593750596046448,
|
|
"reward_std": 0.3606441020965576,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003001186763867736,
|
|
"sampling/sampling_logp_difference/max": 0.8754743337631226,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.0393989086151123,
|
|
"sampling/importance_sampling_ratio/max": 1.4823861122131348,
|
|
"kl": 0.003193561267835321,
|
|
"entropy": 0.05215287418104708,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.576893324032426,
|
|
"epoch": 0.002109375,
|
|
"step": 81
|
|
},
|
|
{
|
|
"loss": 0.32682687044143677,
|
|
"grad_norm": 3.2923216819763184,
|
|
"learning_rate": 3.775862068965517e-07,
|
|
"num_tokens": 890737.0,
|
|
"completions/mean_length": 579.125,
|
|
"completions/min_length": 244.0,
|
|
"completions/max_length": 934.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 262.5,
|
|
"completions/min_terminated_length": 244.0,
|
|
"completions/max_terminated_length": 291.0,
|
|
"tools/call_frequency": 14.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.41874998807907104,
|
|
"rewards/reward_func/std": 0.33602336049079895,
|
|
"reward": 0.41874998807907104,
|
|
"reward_std": 0.33602336049079895,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002651141257956624,
|
|
"sampling/sampling_logp_difference/max": 0.5942459106445312,
|
|
"sampling/importance_sampling_ratio/min": 0.5984089970588684,
|
|
"sampling/importance_sampling_ratio/mean": 1.065975546836853,
|
|
"sampling/importance_sampling_ratio/max": 2.441396713256836,
|
|
"kl": 0.0018357197695877403,
|
|
"entropy": 0.05130588042084128,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.189323069527745,
|
|
"epoch": 0.0021354166666666665,
|
|
"step": 82
|
|
},
|
|
{
|
|
"loss": 0.09406017512083054,
|
|
"grad_norm": 4.016912937164307,
|
|
"learning_rate": 3.758620689655172e-07,
|
|
"num_tokens": 900123.0,
|
|
"completions/mean_length": 486.875,
|
|
"completions/min_length": 208.0,
|
|
"completions/max_length": 992.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 331.3333435058594,
|
|
"completions/min_terminated_length": 208.0,
|
|
"completions/max_terminated_length": 886.0,
|
|
"tools/call_frequency": 11.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.0920138955116272,
|
|
"rewards/reward_func/std": 0.22755232453346252,
|
|
"reward": 0.0920138955116272,
|
|
"reward_std": 0.2275523543357849,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0026918784715235233,
|
|
"sampling/sampling_logp_difference/max": 0.4412808418273926,
|
|
"sampling/importance_sampling_ratio/min": 0.5825228095054626,
|
|
"sampling/importance_sampling_ratio/mean": 1.0555436611175537,
|
|
"sampling/importance_sampling_ratio/max": 1.7297953367233276,
|
|
"kl": 0.003282440027760458,
|
|
"entropy": 0.05685475387144834,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.532877545803785,
|
|
"epoch": 0.0021614583333333334,
|
|
"step": 83
|
|
},
|
|
{
|
|
"loss": 0.3641356825828552,
|
|
"grad_norm": 2.167774200439453,
|
|
"learning_rate": 3.7413793103448276e-07,
|
|
"num_tokens": 910359.0,
|
|
"completions/mean_length": 594.0,
|
|
"completions/min_length": 182.0,
|
|
"completions/max_length": 944.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 548.7142944335938,
|
|
"completions/min_terminated_length": 182.0,
|
|
"completions/max_terminated_length": 944.0,
|
|
"tools/call_frequency": 14.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.22708334028720856,
|
|
"rewards/reward_func/std": 0.320365309715271,
|
|
"reward": 0.22708334028720856,
|
|
"reward_std": 0.320365309715271,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002398031996563077,
|
|
"sampling/sampling_logp_difference/max": 0.7211248874664307,
|
|
"sampling/importance_sampling_ratio/min": 0.41798585653305054,
|
|
"sampling/importance_sampling_ratio/mean": 0.7715437412261963,
|
|
"sampling/importance_sampling_ratio/max": 1.0889952182769775,
|
|
"kl": 0.002181519605073845,
|
|
"entropy": 0.053286053938791156,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.118091953918338,
|
|
"epoch": 0.0021875,
|
|
"step": 84
|
|
},
|
|
{
|
|
"loss": -0.13569596409797668,
|
|
"grad_norm": 4.3375701904296875,
|
|
"learning_rate": 3.7241379310344827e-07,
|
|
"num_tokens": 921889.0,
|
|
"completions/mean_length": 755.5,
|
|
"completions/min_length": 279.0,
|
|
"completions/max_length": 934.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 588.75,
|
|
"completions/min_terminated_length": 279.0,
|
|
"completions/max_terminated_length": 925.0,
|
|
"tools/call_frequency": 18.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21458333730697632,
|
|
"rewards/reward_func/std": 0.26415297389030457,
|
|
"reward": 0.21458333730697632,
|
|
"reward_std": 0.2641529440879822,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019059869227930903,
|
|
"sampling/sampling_logp_difference/max": 0.5358269214630127,
|
|
"sampling/importance_sampling_ratio/min": 0.2943079471588135,
|
|
"sampling/importance_sampling_ratio/mean": 1.032212257385254,
|
|
"sampling/importance_sampling_ratio/max": 1.7351746559143066,
|
|
"kl": 0.0009725792579047265,
|
|
"entropy": 0.03484517044853419,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.34490535594523,
|
|
"epoch": 0.0022135416666666666,
|
|
"step": 85
|
|
},
|
|
{
|
|
"loss": 0.4627648591995239,
|
|
"grad_norm": 2.5416181087493896,
|
|
"learning_rate": 3.706896551724138e-07,
|
|
"num_tokens": 931248.0,
|
|
"completions/mean_length": 485.25,
|
|
"completions/min_length": 152.0,
|
|
"completions/max_length": 947.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 340.66668701171875,
|
|
"completions/min_terminated_length": 152.0,
|
|
"completions/max_terminated_length": 947.0,
|
|
"tools/call_frequency": 11.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3375000059604645,
|
|
"rewards/reward_func/std": 0.39005494117736816,
|
|
"reward": 0.3375000059604645,
|
|
"reward_std": 0.39005494117736816,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0021875789389014244,
|
|
"sampling/sampling_logp_difference/max": 0.47338247299194336,
|
|
"sampling/importance_sampling_ratio/min": 0.633094310760498,
|
|
"sampling/importance_sampling_ratio/mean": 0.9081072807312012,
|
|
"sampling/importance_sampling_ratio/max": 1.252740740776062,
|
|
"kl": 0.0017056657684406673,
|
|
"entropy": 0.049412118503823876,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.536534506827593,
|
|
"epoch": 0.0022395833333333334,
|
|
"step": 86
|
|
},
|
|
{
|
|
"loss": 0.32607507705688477,
|
|
"grad_norm": 15.590494155883789,
|
|
"learning_rate": 3.689655172413793e-07,
|
|
"num_tokens": 941287.0,
|
|
"completions/mean_length": 569.75,
|
|
"completions/min_length": 183.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 451.16668701171875,
|
|
"completions/min_terminated_length": 183.0,
|
|
"completions/max_terminated_length": 931.0,
|
|
"tools/call_frequency": 13.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18013392388820648,
|
|
"rewards/reward_func/std": 0.35024648904800415,
|
|
"reward": 0.18013392388820648,
|
|
"reward_std": 0.35024648904800415,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002008522627875209,
|
|
"sampling/sampling_logp_difference/max": 0.43998265266418457,
|
|
"sampling/importance_sampling_ratio/min": 0.16912336647510529,
|
|
"sampling/importance_sampling_ratio/mean": 1.028156042098999,
|
|
"sampling/importance_sampling_ratio/max": 1.4545624256134033,
|
|
"kl": 0.0031245873469742946,
|
|
"entropy": 0.03804778202902526,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.992027828469872,
|
|
"epoch": 0.002265625,
|
|
"step": 87
|
|
},
|
|
{
|
|
"loss": 0.2087986171245575,
|
|
"grad_norm": 4.229395866394043,
|
|
"learning_rate": 3.6724137931034485e-07,
|
|
"num_tokens": 950742.0,
|
|
"completions/mean_length": 496.625,
|
|
"completions/min_length": 166.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 243.40000915527344,
|
|
"completions/min_terminated_length": 166.0,
|
|
"completions/max_terminated_length": 324.0,
|
|
"tools/call_frequency": 11.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2447916567325592,
|
|
"rewards/reward_func/std": 0.402051717042923,
|
|
"reward": 0.2447916567325592,
|
|
"reward_std": 0.402051717042923,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0034454953856766224,
|
|
"sampling/sampling_logp_difference/max": 0.773344874382019,
|
|
"sampling/importance_sampling_ratio/min": 0.3816334009170532,
|
|
"sampling/importance_sampling_ratio/mean": 1.3282134532928467,
|
|
"sampling/importance_sampling_ratio/max": 2.127211093902588,
|
|
"kl": 0.0011863803392770933,
|
|
"entropy": 0.057057317753788084,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.06111790984869,
|
|
"epoch": 0.0022916666666666667,
|
|
"step": 88
|
|
},
|
|
{
|
|
"loss": 0.36389636993408203,
|
|
"grad_norm": 5.7996087074279785,
|
|
"learning_rate": 3.6551724137931036e-07,
|
|
"num_tokens": 960825.0,
|
|
"completions/mean_length": 575.25,
|
|
"completions/min_length": 202.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 243.5,
|
|
"completions/min_terminated_length": 202.0,
|
|
"completions/max_terminated_length": 308.0,
|
|
"tools/call_frequency": 14.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3342013955116272,
|
|
"rewards/reward_func/std": 0.3647095561027527,
|
|
"reward": 0.3342013955116272,
|
|
"reward_std": 0.3647095859050751,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003254256909713149,
|
|
"sampling/sampling_logp_difference/max": 0.9900604486465454,
|
|
"sampling/importance_sampling_ratio/min": 0.09745648503303528,
|
|
"sampling/importance_sampling_ratio/mean": 0.6431615948677063,
|
|
"sampling/importance_sampling_ratio/max": 1.0968931913375854,
|
|
"kl": 0.0023543541592516704,
|
|
"entropy": 0.045104061835445464,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.040268735960126,
|
|
"epoch": 0.0023177083333333335,
|
|
"step": 89
|
|
},
|
|
{
|
|
"loss": 0.1472841203212738,
|
|
"grad_norm": 3.4667551517486572,
|
|
"learning_rate": 3.6379310344827587e-07,
|
|
"num_tokens": 970963.0,
|
|
"completions/mean_length": 582.125,
|
|
"completions/min_length": 146.0,
|
|
"completions/max_length": 945.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 368.3999938964844,
|
|
"completions/min_terminated_length": 146.0,
|
|
"completions/max_terminated_length": 913.0,
|
|
"tools/call_frequency": 13.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16562500596046448,
|
|
"rewards/reward_func/std": 0.3153619170188904,
|
|
"reward": 0.16562500596046448,
|
|
"reward_std": 0.315361887216568,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002721753204241395,
|
|
"sampling/sampling_logp_difference/max": 1.1265720129013062,
|
|
"sampling/importance_sampling_ratio/min": 0.5274291038513184,
|
|
"sampling/importance_sampling_ratio/mean": 0.8270600438117981,
|
|
"sampling/importance_sampling_ratio/max": 1.1990267038345337,
|
|
"kl": 0.0012024851039313944,
|
|
"entropy": 0.046692353207618,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.417876783758402,
|
|
"epoch": 0.00234375,
|
|
"step": 90
|
|
},
|
|
{
|
|
"loss": 0.13428398966789246,
|
|
"grad_norm": 1.6078310012817383,
|
|
"learning_rate": 3.620689655172414e-07,
|
|
"num_tokens": 980924.0,
|
|
"completions/mean_length": 559.75,
|
|
"completions/min_length": 211.0,
|
|
"completions/max_length": 923.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 354.20001220703125,
|
|
"completions/min_terminated_length": 211.0,
|
|
"completions/max_terminated_length": 833.0,
|
|
"tools/call_frequency": 14.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16006945073604584,
|
|
"rewards/reward_func/std": 0.31002479791641235,
|
|
"reward": 0.16006945073604584,
|
|
"reward_std": 0.31002482771873474,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0031620135996490717,
|
|
"sampling/sampling_logp_difference/max": 0.48098301887512207,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.5790830850601196,
|
|
"sampling/importance_sampling_ratio/max": 1.0290824174880981,
|
|
"kl": 0.0017000705720420228,
|
|
"entropy": 0.06004357093479484,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.400904923677444,
|
|
"epoch": 0.0023697916666666667,
|
|
"step": 91
|
|
},
|
|
{
|
|
"loss": 0.16187749803066254,
|
|
"grad_norm": 6.078117370605469,
|
|
"learning_rate": 3.603448275862069e-07,
|
|
"num_tokens": 991019.0,
|
|
"completions/mean_length": 576.875,
|
|
"completions/min_length": 228.0,
|
|
"completions/max_length": 912.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 401.6000061035156,
|
|
"completions/min_terminated_length": 228.0,
|
|
"completions/max_terminated_length": 912.0,
|
|
"tools/call_frequency": 14.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.24375000596046448,
|
|
"rewards/reward_func/std": 0.29177719354629517,
|
|
"reward": 0.24375000596046448,
|
|
"reward_std": 0.29177719354629517,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0027243157383054495,
|
|
"sampling/sampling_logp_difference/max": 0.3980226516723633,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.131995439529419,
|
|
"sampling/importance_sampling_ratio/max": 2.5091092586517334,
|
|
"kl": 0.001545590841487865,
|
|
"entropy": 0.054546710569411516,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.501275930553675,
|
|
"epoch": 0.002395833333333333,
|
|
"step": 92
|
|
},
|
|
{
|
|
"loss": -0.0520855076611042,
|
|
"grad_norm": 4.095066547393799,
|
|
"learning_rate": 3.586206896551724e-07,
|
|
"num_tokens": 1000445.0,
|
|
"completions/mean_length": 492.5,
|
|
"completions/min_length": 217.0,
|
|
"completions/max_length": 946.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 228.40000915527344,
|
|
"completions/min_terminated_length": 217.0,
|
|
"completions/max_terminated_length": 245.0,
|
|
"tools/call_frequency": 11.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.15669643878936768,
|
|
"rewards/reward_func/std": 0.3562869131565094,
|
|
"reward": 0.15669643878936768,
|
|
"reward_std": 0.3562869131565094,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003694097977131605,
|
|
"sampling/sampling_logp_difference/max": 1.0769623517990112,
|
|
"sampling/importance_sampling_ratio/min": 0.10566303133964539,
|
|
"sampling/importance_sampling_ratio/mean": 0.6034933924674988,
|
|
"sampling/importance_sampling_ratio/max": 1.504448413848877,
|
|
"kl": 0.0038821143498353194,
|
|
"entropy": 0.05784318596124649,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.77889876253903,
|
|
"epoch": 0.002421875,
|
|
"step": 93
|
|
},
|
|
{
|
|
"loss": 0.3161385655403137,
|
|
"grad_norm": 3.376772403717041,
|
|
"learning_rate": 3.568965517241379e-07,
|
|
"num_tokens": 1011194.0,
|
|
"completions/mean_length": 658.125,
|
|
"completions/min_length": 140.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 400.75,
|
|
"completions/min_terminated_length": 140.0,
|
|
"completions/max_terminated_length": 890.0,
|
|
"tools/call_frequency": 15.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18715277314186096,
|
|
"rewards/reward_func/std": 0.3231341242790222,
|
|
"reward": 0.18715277314186096,
|
|
"reward_std": 0.32313409447669983,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020105040166527033,
|
|
"sampling/sampling_logp_difference/max": 0.6759900450706482,
|
|
"sampling/importance_sampling_ratio/min": 0.49338841438293457,
|
|
"sampling/importance_sampling_ratio/mean": 0.9529753923416138,
|
|
"sampling/importance_sampling_ratio/max": 1.938850998878479,
|
|
"kl": 0.0012230866468598833,
|
|
"entropy": 0.03659372485708445,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.550245963037014,
|
|
"epoch": 0.002447916666666667,
|
|
"step": 94
|
|
},
|
|
{
|
|
"loss": -0.21851398050785065,
|
|
"grad_norm": 1.4297304153442383,
|
|
"learning_rate": 3.551724137931034e-07,
|
|
"num_tokens": 1022635.0,
|
|
"completions/mean_length": 745.25,
|
|
"completions/min_length": 196.0,
|
|
"completions/max_length": 947.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 559.5,
|
|
"completions/min_terminated_length": 196.0,
|
|
"completions/max_terminated_length": 900.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06562499701976776,
|
|
"rewards/reward_func/std": 0.17875437438488007,
|
|
"reward": 0.06562499701976776,
|
|
"reward_std": 0.17875435948371887,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016958089545369148,
|
|
"sampling/sampling_logp_difference/max": 1.2012001276016235,
|
|
"sampling/importance_sampling_ratio/min": 0.27445781230926514,
|
|
"sampling/importance_sampling_ratio/mean": 0.7518707513809204,
|
|
"sampling/importance_sampling_ratio/max": 1.2732913494110107,
|
|
"kl": 0.0006457657077589829,
|
|
"entropy": 0.01979198312619701,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.562746416777372,
|
|
"epoch": 0.0024739583333333332,
|
|
"step": 95
|
|
},
|
|
{
|
|
"loss": 0.1475478559732437,
|
|
"grad_norm": 3.570300579071045,
|
|
"learning_rate": 3.534482758620689e-07,
|
|
"num_tokens": 1033332.0,
|
|
"completions/mean_length": 652.0,
|
|
"completions/min_length": 210.0,
|
|
"completions/max_length": 938.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 567.5,
|
|
"completions/min_terminated_length": 210.0,
|
|
"completions/max_terminated_length": 938.0,
|
|
"tools/call_frequency": 15.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.15461309254169464,
|
|
"rewards/reward_func/std": 0.3308669924736023,
|
|
"reward": 0.15461309254169464,
|
|
"reward_std": 0.3308669924736023,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0012970935786142945,
|
|
"sampling/sampling_logp_difference/max": 0.4873185157775879,
|
|
"sampling/importance_sampling_ratio/min": 0.7817084789276123,
|
|
"sampling/importance_sampling_ratio/mean": 1.210066795349121,
|
|
"sampling/importance_sampling_ratio/max": 2.5019965171813965,
|
|
"kl": 0.0008418744928349042,
|
|
"entropy": 0.031776201649336144,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.470612047240138,
|
|
"epoch": 0.0025,
|
|
"step": 96
|
|
},
|
|
{
|
|
"loss": 0.44571396708488464,
|
|
"grad_norm": 1.0614652633666992,
|
|
"learning_rate": 3.517241379310344e-07,
|
|
"num_tokens": 1045527.0,
|
|
"completions/mean_length": 838.125,
|
|
"completions/min_length": 223.0,
|
|
"completions/max_length": 997.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 814.6666870117188,
|
|
"completions/min_terminated_length": 223.0,
|
|
"completions/max_terminated_length": 997.0,
|
|
"tools/call_frequency": 19.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1822916716337204,
|
|
"rewards/reward_func/std": 0.2584461271762848,
|
|
"reward": 0.1822916716337204,
|
|
"reward_std": 0.2584461271762848,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013216750230640173,
|
|
"sampling/sampling_logp_difference/max": 0.4954681396484375,
|
|
"sampling/importance_sampling_ratio/min": 0.4966128170490265,
|
|
"sampling/importance_sampling_ratio/mean": 1.2125492095947266,
|
|
"sampling/importance_sampling_ratio/max": 2.2518022060394287,
|
|
"kl": 0.0007104939368218766,
|
|
"entropy": 0.02161255502142012,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.417735720053315,
|
|
"epoch": 0.0025260416666666665,
|
|
"step": 97
|
|
},
|
|
{
|
|
"loss": 0.4199284315109253,
|
|
"grad_norm": 1.259472131729126,
|
|
"learning_rate": 3.5e-07,
|
|
"num_tokens": 1056361.0,
|
|
"completions/mean_length": 668.875,
|
|
"completions/min_length": 98.0,
|
|
"completions/max_length": 941.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 636.2857666015625,
|
|
"completions/min_terminated_length": 98.0,
|
|
"completions/max_terminated_length": 941.0,
|
|
"tools/call_frequency": 16.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13124999403953552,
|
|
"rewards/reward_func/std": 0.2034303992986679,
|
|
"reward": 0.13124999403953552,
|
|
"reward_std": 0.2034303992986679,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019448194652795792,
|
|
"sampling/sampling_logp_difference/max": 0.6848430037498474,
|
|
"sampling/importance_sampling_ratio/min": 0.24199728667736053,
|
|
"sampling/importance_sampling_ratio/mean": 1.1141021251678467,
|
|
"sampling/importance_sampling_ratio/max": 2.4243462085723877,
|
|
"kl": 0.0005638301938688528,
|
|
"entropy": 0.03213238145690411,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.30503617413342,
|
|
"epoch": 0.0025520833333333333,
|
|
"step": 98
|
|
},
|
|
{
|
|
"loss": 0.08396437764167786,
|
|
"grad_norm": 3.8016769886016846,
|
|
"learning_rate": 3.482758620689655e-07,
|
|
"num_tokens": 1066989.0,
|
|
"completions/mean_length": 644.0,
|
|
"completions/min_length": 192.0,
|
|
"completions/max_length": 927.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 564.0,
|
|
"completions/min_terminated_length": 192.0,
|
|
"completions/max_terminated_length": 927.0,
|
|
"tools/call_frequency": 15.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.11874999850988388,
|
|
"rewards/reward_func/std": 0.3228638768196106,
|
|
"reward": 0.11874999850988388,
|
|
"reward_std": 0.3228638470172882,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015953953843563795,
|
|
"sampling/sampling_logp_difference/max": 0.4672126770019531,
|
|
"sampling/importance_sampling_ratio/min": 0.5818261504173279,
|
|
"sampling/importance_sampling_ratio/mean": 1.1711058616638184,
|
|
"sampling/importance_sampling_ratio/max": 1.7215070724487305,
|
|
"kl": 0.0011468135297150184,
|
|
"entropy": 0.032332405040506274,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.967326793819666,
|
|
"epoch": 0.002578125,
|
|
"step": 99
|
|
},
|
|
{
|
|
"loss": -0.18173733353614807,
|
|
"grad_norm": 2.7329866886138916,
|
|
"learning_rate": 3.4655172413793105e-07,
|
|
"num_tokens": 1077727.0,
|
|
"completions/mean_length": 656.625,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 947.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 565.8333740234375,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 925.0,
|
|
"tools/call_frequency": 15.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05000000074505806,
|
|
"rewards/reward_func/std": 0.13887301087379456,
|
|
"reward": 0.05000000074505806,
|
|
"reward_std": 0.13887301087379456,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016767301131039858,
|
|
"sampling/sampling_logp_difference/max": 0.8849389553070068,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.0322985649108887,
|
|
"sampling/importance_sampling_ratio/max": 1.7402434349060059,
|
|
"kl": 0.000279801898614096,
|
|
"entropy": 0.03210699529154226,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.410941718146205,
|
|
"epoch": 0.0026041666666666665,
|
|
"step": 100
|
|
},
|
|
{
|
|
"loss": -0.003773435950279236,
|
|
"grad_norm": 2.6353883743286133,
|
|
"learning_rate": 3.4482758620689656e-07,
|
|
"num_tokens": 1089486.0,
|
|
"completions/mean_length": 785.25,
|
|
"completions/min_length": 330.0,
|
|
"completions/max_length": 981.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 642.5,
|
|
"completions/min_terminated_length": 330.0,
|
|
"completions/max_terminated_length": 928.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06875000149011612,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.06875000149011612,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.00230760988779366,
|
|
"sampling/sampling_logp_difference/max": 0.43779075145721436,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.75773024559021,
|
|
"sampling/importance_sampling_ratio/max": 1.570510983467102,
|
|
"kl": 0.00030970063835411565,
|
|
"entropy": 0.041491779615171254,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.102560376748443,
|
|
"epoch": 0.0026302083333333334,
|
|
"step": 101
|
|
},
|
|
{
|
|
"loss": 0.2676636576652527,
|
|
"grad_norm": 1.5777575969696045,
|
|
"learning_rate": 3.4310344827586207e-07,
|
|
"num_tokens": 1101655.0,
|
|
"completions/mean_length": 835.875,
|
|
"completions/min_length": 279.0,
|
|
"completions/max_length": 942.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 706.6666870117188,
|
|
"completions/min_terminated_length": 279.0,
|
|
"completions/max_terminated_length": 925.0,
|
|
"tools/call_frequency": 19.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.07187499850988388,
|
|
"rewards/reward_func/std": 0.061871837824583054,
|
|
"reward": 0.07187499850988388,
|
|
"reward_std": 0.06187184527516365,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001240467419847846,
|
|
"sampling/sampling_logp_difference/max": 0.2808610796928406,
|
|
"sampling/importance_sampling_ratio/min": 0.4135251045227051,
|
|
"sampling/importance_sampling_ratio/mean": 1.0815691947937012,
|
|
"sampling/importance_sampling_ratio/max": 1.8918157815933228,
|
|
"kl": 0.00021856003240827704,
|
|
"entropy": 0.020778865728061646,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.093903051689267,
|
|
"epoch": 0.00265625,
|
|
"step": 102
|
|
},
|
|
{
|
|
"loss": -0.1673833578824997,
|
|
"grad_norm": 6.115754127502441,
|
|
"learning_rate": 3.413793103448276e-07,
|
|
"num_tokens": 1111375.0,
|
|
"completions/mean_length": 529.75,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 916.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 303.20001220703125,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 630.0,
|
|
"tools/call_frequency": 12.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.32055556774139404,
|
|
"rewards/reward_func/std": 0.3387969434261322,
|
|
"reward": 0.32055556774139404,
|
|
"reward_std": 0.3387969136238098,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020829110872000456,
|
|
"sampling/sampling_logp_difference/max": 0.4754652976989746,
|
|
"sampling/importance_sampling_ratio/min": 0.7333513498306274,
|
|
"sampling/importance_sampling_ratio/mean": 1.0969946384429932,
|
|
"sampling/importance_sampling_ratio/max": 1.5877243280410767,
|
|
"kl": 0.0022351465263454884,
|
|
"entropy": 0.04086584266042337,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.049891103059053,
|
|
"epoch": 0.0026822916666666666,
|
|
"step": 103
|
|
},
|
|
{
|
|
"loss": 0.02801184169948101,
|
|
"grad_norm": 4.5080976486206055,
|
|
"learning_rate": 3.396551724137931e-07,
|
|
"num_tokens": 1121436.0,
|
|
"completions/mean_length": 572.375,
|
|
"completions/min_length": 176.0,
|
|
"completions/max_length": 961.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 528.1428833007812,
|
|
"completions/min_terminated_length": 176.0,
|
|
"completions/max_terminated_length": 961.0,
|
|
"tools/call_frequency": 13.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.07083333283662796,
|
|
"rewards/reward_func/std": 0.2519369423389435,
|
|
"reward": 0.07083333283662796,
|
|
"reward_std": 0.2519369423389435,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0026417130138725042,
|
|
"sampling/sampling_logp_difference/max": 0.8754712343215942,
|
|
"sampling/importance_sampling_ratio/min": 0.40278589725494385,
|
|
"sampling/importance_sampling_ratio/mean": 0.9573386907577515,
|
|
"sampling/importance_sampling_ratio/max": 1.9268139600753784,
|
|
"kl": 0.001198325163386471,
|
|
"entropy": 0.04489107127301395,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.114700522273779,
|
|
"epoch": 0.0027083333333333334,
|
|
"step": 104
|
|
},
|
|
{
|
|
"loss": 0.19745346903800964,
|
|
"grad_norm": 2.87408447265625,
|
|
"learning_rate": 3.379310344827586e-07,
|
|
"num_tokens": 1130813.0,
|
|
"completions/mean_length": 487.125,
|
|
"completions/min_length": 160.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 343.66668701171875,
|
|
"completions/min_terminated_length": 160.0,
|
|
"completions/max_terminated_length": 900.0,
|
|
"tools/call_frequency": 12.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2274147868156433,
|
|
"rewards/reward_func/std": 0.339668869972229,
|
|
"reward": 0.2274147868156433,
|
|
"reward_std": 0.339668869972229,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002187444129958749,
|
|
"sampling/sampling_logp_difference/max": 0.31093132495880127,
|
|
"sampling/importance_sampling_ratio/min": 0.7228125929832458,
|
|
"sampling/importance_sampling_ratio/mean": 0.9490094184875488,
|
|
"sampling/importance_sampling_ratio/max": 1.19515061378479,
|
|
"kl": 0.001829710136007634,
|
|
"entropy": 0.04906006238888949,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.805498618632555,
|
|
"epoch": 0.002734375,
|
|
"step": 105
|
|
},
|
|
{
|
|
"loss": 0.36593639850616455,
|
|
"grad_norm": 1.7658203840255737,
|
|
"learning_rate": 3.362068965517241e-07,
|
|
"num_tokens": 1142215.0,
|
|
"completions/mean_length": 740.75,
|
|
"completions/min_length": 214.0,
|
|
"completions/max_length": 951.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 648.6000366210938,
|
|
"completions/min_terminated_length": 214.0,
|
|
"completions/max_terminated_length": 941.0,
|
|
"tools/call_frequency": 17.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17083333432674408,
|
|
"rewards/reward_func/std": 0.25401535630226135,
|
|
"reward": 0.17083333432674408,
|
|
"reward_std": 0.25401535630226135,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002073743147775531,
|
|
"sampling/sampling_logp_difference/max": 0.8482702970504761,
|
|
"sampling/importance_sampling_ratio/min": 0.4761240482330322,
|
|
"sampling/importance_sampling_ratio/mean": 1.005946397781372,
|
|
"sampling/importance_sampling_ratio/max": 1.9766294956207275,
|
|
"kl": 0.0011398474189263652,
|
|
"entropy": 0.03656519704964012,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.845880715176463,
|
|
"epoch": 0.0027604166666666667,
|
|
"step": 106
|
|
},
|
|
{
|
|
"loss": 0.7766353487968445,
|
|
"grad_norm": 2.3821632862091064,
|
|
"learning_rate": 3.3448275862068966e-07,
|
|
"num_tokens": 1152378.0,
|
|
"completions/mean_length": 585.625,
|
|
"completions/min_length": 206.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 382.8000183105469,
|
|
"completions/min_terminated_length": 206.0,
|
|
"completions/max_terminated_length": 921.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23541666567325592,
|
|
"rewards/reward_func/std": 0.2704255282878876,
|
|
"reward": 0.23541666567325592,
|
|
"reward_std": 0.2704255282878876,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022846381179988384,
|
|
"sampling/sampling_logp_difference/max": 0.6863491535186768,
|
|
"sampling/importance_sampling_ratio/min": 0.27012553811073303,
|
|
"sampling/importance_sampling_ratio/mean": 1.197967767715454,
|
|
"sampling/importance_sampling_ratio/max": 2.2859175205230713,
|
|
"kl": 0.001589251132827485,
|
|
"entropy": 0.04594969714526087,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.474369483068585,
|
|
"epoch": 0.0027864583333333335,
|
|
"step": 107
|
|
},
|
|
{
|
|
"loss": 0.28064143657684326,
|
|
"grad_norm": 1.750715970993042,
|
|
"learning_rate": 3.3275862068965517e-07,
|
|
"num_tokens": 1163745.0,
|
|
"completions/mean_length": 736.75,
|
|
"completions/min_length": 235.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 576.25,
|
|
"completions/min_terminated_length": 235.0,
|
|
"completions/max_terminated_length": 914.0,
|
|
"tools/call_frequency": 18.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.20510418713092804,
|
|
"rewards/reward_func/std": 0.28915947675704956,
|
|
"reward": 0.20510418713092804,
|
|
"reward_std": 0.28915947675704956,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016862296033650637,
|
|
"sampling/sampling_logp_difference/max": 0.7273367047309875,
|
|
"sampling/importance_sampling_ratio/min": 0.4828726053237915,
|
|
"sampling/importance_sampling_ratio/mean": 0.8734228610992432,
|
|
"sampling/importance_sampling_ratio/max": 1.4140714406967163,
|
|
"kl": 0.0007017806437943364,
|
|
"entropy": 0.036348951398395,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.314145229756832,
|
|
"epoch": 0.0028125,
|
|
"step": 108
|
|
},
|
|
{
|
|
"loss": 0.05474608391523361,
|
|
"grad_norm": 2.4352564811706543,
|
|
"learning_rate": 3.310344827586207e-07,
|
|
"num_tokens": 1173634.0,
|
|
"completions/mean_length": 550.375,
|
|
"completions/min_length": 141.0,
|
|
"completions/max_length": 980.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 172.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"completions/max_terminated_length": 212.0,
|
|
"tools/call_frequency": 12.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.0647321492433548,
|
|
"rewards/reward_func/std": 0.23550115525722504,
|
|
"reward": 0.0647321492433548,
|
|
"reward_std": 0.23550112545490265,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018824965227395296,
|
|
"sampling/sampling_logp_difference/max": 0.643399715423584,
|
|
"sampling/importance_sampling_ratio/min": 0.46323758363723755,
|
|
"sampling/importance_sampling_ratio/mean": 1.1476387977600098,
|
|
"sampling/importance_sampling_ratio/max": 1.7960578203201294,
|
|
"kl": 0.0021688767219529836,
|
|
"entropy": 0.040713865077123046,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.527055049315095,
|
|
"epoch": 0.0028385416666666667,
|
|
"step": 109
|
|
},
|
|
{
|
|
"loss": -0.017282232642173767,
|
|
"grad_norm": 1.8323973417282104,
|
|
"learning_rate": 3.293103448275862e-07,
|
|
"num_tokens": 1185762.0,
|
|
"completions/mean_length": 829.75,
|
|
"completions/min_length": 241.0,
|
|
"completions/max_length": 974.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 754.25,
|
|
"completions/min_terminated_length": 241.0,
|
|
"completions/max_terminated_length": 941.0,
|
|
"tools/call_frequency": 18.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.137648805975914,
|
|
"rewards/reward_func/std": 0.18547423183918,
|
|
"reward": 0.137648805975914,
|
|
"reward_std": 0.18547423183918,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.00146865239366889,
|
|
"sampling/sampling_logp_difference/max": 0.4272884726524353,
|
|
"sampling/importance_sampling_ratio/min": 0.4101531207561493,
|
|
"sampling/importance_sampling_ratio/mean": 0.8662111759185791,
|
|
"sampling/importance_sampling_ratio/max": 2.1287269592285156,
|
|
"kl": 0.0008217824747589475,
|
|
"entropy": 0.02545974589884281,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.16356516815722,
|
|
"epoch": 0.002864583333333333,
|
|
"step": 110
|
|
},
|
|
{
|
|
"loss": 0.09654361009597778,
|
|
"grad_norm": 3.9607038497924805,
|
|
"learning_rate": 3.275862068965517e-07,
|
|
"num_tokens": 1196469.0,
|
|
"completions/mean_length": 653.5,
|
|
"completions/min_length": 168.0,
|
|
"completions/max_length": 947.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 393.5,
|
|
"completions/min_terminated_length": 168.0,
|
|
"completions/max_terminated_length": 914.0,
|
|
"tools/call_frequency": 15.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16111111640930176,
|
|
"rewards/reward_func/std": 0.21396875381469727,
|
|
"reward": 0.16111111640930176,
|
|
"reward_std": 0.21396875381469727,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020886205602437258,
|
|
"sampling/sampling_logp_difference/max": 0.47156333923339844,
|
|
"sampling/importance_sampling_ratio/min": 0.5459579825401306,
|
|
"sampling/importance_sampling_ratio/mean": 1.365042805671692,
|
|
"sampling/importance_sampling_ratio/max": 1.8374704122543335,
|
|
"kl": 0.001211339281326218,
|
|
"entropy": 0.03619563241954893,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 14.21070434525609,
|
|
"epoch": 0.002890625,
|
|
"step": 111
|
|
},
|
|
{
|
|
"loss": 0.1050049215555191,
|
|
"grad_norm": 1.801943302154541,
|
|
"learning_rate": 3.258620689655172e-07,
|
|
"num_tokens": 1207208.0,
|
|
"completions/mean_length": 656.25,
|
|
"completions/min_length": 144.0,
|
|
"completions/max_length": 963.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 382.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"completions/max_terminated_length": 932.0,
|
|
"tools/call_frequency": 14.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.09947916865348816,
|
|
"rewards/reward_func/std": 0.2698143720626831,
|
|
"reward": 0.09947916865348816,
|
|
"reward_std": 0.2698143422603607,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016243589343503118,
|
|
"sampling/sampling_logp_difference/max": 0.36019444465637207,
|
|
"sampling/importance_sampling_ratio/min": 0.3685455024242401,
|
|
"sampling/importance_sampling_ratio/mean": 0.877813458442688,
|
|
"sampling/importance_sampling_ratio/max": 1.2708152532577515,
|
|
"kl": 0.0008343038774416556,
|
|
"entropy": 0.029123721120413393,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.768606916069984,
|
|
"epoch": 0.002916666666666667,
|
|
"step": 112
|
|
},
|
|
{
|
|
"loss": 0.44175755977630615,
|
|
"grad_norm": 2.720227003097534,
|
|
"learning_rate": 3.241379310344827e-07,
|
|
"num_tokens": 1216548.0,
|
|
"completions/mean_length": 483.125,
|
|
"completions/min_length": 183.0,
|
|
"completions/max_length": 1004.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 327.0,
|
|
"completions/min_terminated_length": 183.0,
|
|
"completions/max_terminated_length": 778.0,
|
|
"tools/call_frequency": 12.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2760416865348816,
|
|
"rewards/reward_func/std": 0.30454355478286743,
|
|
"reward": 0.2760416865348816,
|
|
"reward_std": 0.30454355478286743,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002974244300276041,
|
|
"sampling/sampling_logp_difference/max": 0.49200010299682617,
|
|
"sampling/importance_sampling_ratio/min": 0.2593773305416107,
|
|
"sampling/importance_sampling_ratio/mean": 0.7866955399513245,
|
|
"sampling/importance_sampling_ratio/max": 1.5099470615386963,
|
|
"kl": 0.002085019879871197,
|
|
"entropy": 0.05230878037400544,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.401849083602428,
|
|
"epoch": 0.002942708333333333,
|
|
"step": 113
|
|
},
|
|
{
|
|
"loss": 0.15870939195156097,
|
|
"grad_norm": 5.393706798553467,
|
|
"learning_rate": 3.224137931034482e-07,
|
|
"num_tokens": 1226870.0,
|
|
"completions/mean_length": 606.25,
|
|
"completions/min_length": 238.0,
|
|
"completions/max_length": 944.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 287.0,
|
|
"completions/min_terminated_length": 238.0,
|
|
"completions/max_terminated_length": 336.0,
|
|
"tools/call_frequency": 14.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16822916269302368,
|
|
"rewards/reward_func/std": 0.23559635877609253,
|
|
"reward": 0.16822916269302368,
|
|
"reward_std": 0.23559635877609253,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0032587535679340363,
|
|
"sampling/sampling_logp_difference/max": 0.9508523941040039,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8946288824081421,
|
|
"sampling/importance_sampling_ratio/max": 2.507766008377075,
|
|
"kl": 0.0010118630284523533,
|
|
"entropy": 0.04696369369048625,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.935637079179287,
|
|
"epoch": 0.00296875,
|
|
"step": 114
|
|
},
|
|
{
|
|
"loss": -0.06677179038524628,
|
|
"grad_norm": 5.862583160400391,
|
|
"learning_rate": 3.2068965517241373e-07,
|
|
"num_tokens": 1235403.0,
|
|
"completions/mean_length": 383.125,
|
|
"completions/min_length": 175.0,
|
|
"completions/max_length": 920.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 205.6666717529297,
|
|
"completions/min_terminated_length": 175.0,
|
|
"completions/max_terminated_length": 277.0,
|
|
"tools/call_frequency": 9.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.08169642090797424,
|
|
"rewards/reward_func/std": 0.3402828276157379,
|
|
"reward": 0.08169642090797424,
|
|
"reward_std": 0.3402828276157379,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0029708249494433403,
|
|
"sampling/sampling_logp_difference/max": 0.5622386932373047,
|
|
"sampling/importance_sampling_ratio/min": 0.30221378803253174,
|
|
"sampling/importance_sampling_ratio/mean": 0.9266810417175293,
|
|
"sampling/importance_sampling_ratio/max": 1.5485930442810059,
|
|
"kl": 0.001808920227517774,
|
|
"entropy": 0.049403826240450144,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.133598230779171,
|
|
"epoch": 0.002994791666666667,
|
|
"step": 115
|
|
},
|
|
{
|
|
"loss": 0.18600031733512878,
|
|
"grad_norm": 2.8649723529815674,
|
|
"learning_rate": 3.1896551724137934e-07,
|
|
"num_tokens": 1245457.0,
|
|
"completions/mean_length": 571.375,
|
|
"completions/min_length": 218.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 377.0,
|
|
"completions/min_terminated_length": 218.0,
|
|
"completions/max_terminated_length": 890.0,
|
|
"tools/call_frequency": 14.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2291666716337204,
|
|
"rewards/reward_func/std": 0.3060241639614105,
|
|
"reward": 0.2291666716337204,
|
|
"reward_std": 0.3060241937637329,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002823794027790427,
|
|
"sampling/sampling_logp_difference/max": 0.9212415218353271,
|
|
"sampling/importance_sampling_ratio/min": 0.5313674211502075,
|
|
"sampling/importance_sampling_ratio/mean": 1.1187288761138916,
|
|
"sampling/importance_sampling_ratio/max": 1.9023722410202026,
|
|
"kl": 0.0015598047502862755,
|
|
"entropy": 0.05225760058965534,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.899680856615305,
|
|
"epoch": 0.0030208333333333333,
|
|
"step": 116
|
|
},
|
|
{
|
|
"loss": 0.07091768085956573,
|
|
"grad_norm": 2.621828556060791,
|
|
"learning_rate": 3.1724137931034485e-07,
|
|
"num_tokens": 1254245.0,
|
|
"completions/mean_length": 414.25,
|
|
"completions/min_length": 200.0,
|
|
"completions/max_length": 923.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 251.1666717529297,
|
|
"completions/min_terminated_length": 200.0,
|
|
"completions/max_terminated_length": 313.0,
|
|
"tools/call_frequency": 9.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3557291626930237,
|
|
"rewards/reward_func/std": 0.3188905119895935,
|
|
"reward": 0.3557291626930237,
|
|
"reward_std": 0.3188905119895935,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003364135045558214,
|
|
"sampling/sampling_logp_difference/max": 0.4412156343460083,
|
|
"sampling/importance_sampling_ratio/min": 0.3753206729888916,
|
|
"sampling/importance_sampling_ratio/mean": 0.8740565180778503,
|
|
"sampling/importance_sampling_ratio/max": 1.3527454137802124,
|
|
"kl": 0.0021807400007674005,
|
|
"entropy": 0.0664672990096733,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.824142830446362,
|
|
"epoch": 0.003046875,
|
|
"step": 117
|
|
},
|
|
{
|
|
"loss": 0.37059861421585083,
|
|
"grad_norm": 3.2843871116638184,
|
|
"learning_rate": 3.1551724137931036e-07,
|
|
"num_tokens": 1263760.0,
|
|
"completions/mean_length": 505.25,
|
|
"completions/min_length": 184.0,
|
|
"completions/max_length": 951.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 358.3333435058594,
|
|
"completions/min_terminated_length": 184.0,
|
|
"completions/max_terminated_length": 918.0,
|
|
"tools/call_frequency": 11.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2447916716337204,
|
|
"rewards/reward_func/std": 0.32197731733322144,
|
|
"reward": 0.2447916716337204,
|
|
"reward_std": 0.32197731733322144,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002230469137430191,
|
|
"sampling/sampling_logp_difference/max": 0.624997615814209,
|
|
"sampling/importance_sampling_ratio/min": 0.41678470373153687,
|
|
"sampling/importance_sampling_ratio/mean": 1.0030547380447388,
|
|
"sampling/importance_sampling_ratio/max": 2.0689284801483154,
|
|
"kl": 0.0012355923086033727,
|
|
"entropy": 0.0423567007528618,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.18518210761249,
|
|
"epoch": 0.0030729166666666665,
|
|
"step": 118
|
|
},
|
|
{
|
|
"loss": -0.17021164298057556,
|
|
"grad_norm": 4.830914497375488,
|
|
"learning_rate": 3.1379310344827587e-07,
|
|
"num_tokens": 1274442.0,
|
|
"completions/mean_length": 649.625,
|
|
"completions/min_length": 177.0,
|
|
"completions/max_length": 927.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 485.6000061035156,
|
|
"completions/min_terminated_length": 177.0,
|
|
"completions/max_terminated_length": 922.0,
|
|
"tools/call_frequency": 14.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.04062500596046448,
|
|
"rewards/reward_func/std": 0.17213860154151917,
|
|
"reward": 0.04062500596046448,
|
|
"reward_std": 0.17213860154151917,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015942062018439174,
|
|
"sampling/sampling_logp_difference/max": 0.422548770904541,
|
|
"sampling/importance_sampling_ratio/min": 0.6595046520233154,
|
|
"sampling/importance_sampling_ratio/mean": 1.1552554368972778,
|
|
"sampling/importance_sampling_ratio/max": 2.3290390968322754,
|
|
"kl": 0.004197305817797314,
|
|
"entropy": 0.04177065857220441,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.098013350740075,
|
|
"epoch": 0.0030989583333333333,
|
|
"step": 119
|
|
},
|
|
{
|
|
"loss": -0.034240517765283585,
|
|
"grad_norm": 2.691129446029663,
|
|
"learning_rate": 3.120689655172414e-07,
|
|
"num_tokens": 1284332.0,
|
|
"completions/mean_length": 550.75,
|
|
"completions/min_length": 148.0,
|
|
"completions/max_length": 978.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 190.5,
|
|
"completions/min_terminated_length": 148.0,
|
|
"completions/max_terminated_length": 215.0,
|
|
"tools/call_frequency": 12.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23050594329833984,
|
|
"rewards/reward_func/std": 0.3801388740539551,
|
|
"reward": 0.23050594329833984,
|
|
"reward_std": 0.3801388442516327,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022434163838624954,
|
|
"sampling/sampling_logp_difference/max": 0.6467037200927734,
|
|
"sampling/importance_sampling_ratio/min": 0.7931464314460754,
|
|
"sampling/importance_sampling_ratio/mean": 1.2336150407791138,
|
|
"sampling/importance_sampling_ratio/max": 2.0688037872314453,
|
|
"kl": 0.0015288488066289574,
|
|
"entropy": 0.04280902200844139,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.91071755811572,
|
|
"epoch": 0.003125,
|
|
"step": 120
|
|
},
|
|
{
|
|
"loss": 0.043542709201574326,
|
|
"grad_norm": 4.81410551071167,
|
|
"learning_rate": 3.103448275862069e-07,
|
|
"num_tokens": 1294443.0,
|
|
"completions/mean_length": 577.625,
|
|
"completions/min_length": 98.0,
|
|
"completions/max_length": 939.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 461.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"completions/max_terminated_length": 939.0,
|
|
"tools/call_frequency": 13.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16874998807907104,
|
|
"rewards/reward_func/std": 0.3207676112651825,
|
|
"reward": 0.16874998807907104,
|
|
"reward_std": 0.3207676410675049,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002132549649104476,
|
|
"sampling/sampling_logp_difference/max": 0.46738100051879883,
|
|
"sampling/importance_sampling_ratio/min": 0.3919413685798645,
|
|
"sampling/importance_sampling_ratio/mean": 0.9637811779975891,
|
|
"sampling/importance_sampling_ratio/max": 1.846157431602478,
|
|
"kl": 0.001004216689580062,
|
|
"entropy": 0.04339109605643898,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.870167758315802,
|
|
"epoch": 0.0031510416666666666,
|
|
"step": 121
|
|
},
|
|
{
|
|
"loss": 0.029441531747579575,
|
|
"grad_norm": 1.7267756462097168,
|
|
"learning_rate": 3.086206896551724e-07,
|
|
"num_tokens": 1306004.0,
|
|
"completions/mean_length": 757.75,
|
|
"completions/min_length": 199.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 479.3333435058594,
|
|
"completions/min_terminated_length": 199.0,
|
|
"completions/max_terminated_length": 905.0,
|
|
"tools/call_frequency": 16.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.12438447028398514,
|
|
"rewards/reward_func/std": 0.1657751053571701,
|
|
"reward": 0.12438447028398514,
|
|
"reward_std": 0.1657751053571701,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014766957610845566,
|
|
"sampling/sampling_logp_difference/max": 0.4200916290283203,
|
|
"sampling/importance_sampling_ratio/min": 0.6608482599258423,
|
|
"sampling/importance_sampling_ratio/mean": 0.9014459252357483,
|
|
"sampling/importance_sampling_ratio/max": 1.1759634017944336,
|
|
"kl": 0.0007242607557600422,
|
|
"entropy": 0.02697584987618029,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.099219648167491,
|
|
"epoch": 0.0031770833333333334,
|
|
"step": 122
|
|
},
|
|
{
|
|
"loss": 0.08653369545936584,
|
|
"grad_norm": 3.567267417907715,
|
|
"learning_rate": 3.068965517241379e-07,
|
|
"num_tokens": 1314168.0,
|
|
"completions/mean_length": 335.75,
|
|
"completions/min_length": 168.0,
|
|
"completions/max_length": 911.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 253.57144165039062,
|
|
"completions/min_terminated_length": 168.0,
|
|
"completions/max_terminated_length": 350.0,
|
|
"tools/call_frequency": 8.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16491477191448212,
|
|
"rewards/reward_func/std": 0.26251542568206787,
|
|
"reward": 0.16491477191448212,
|
|
"reward_std": 0.26251542568206787,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.00538016390055418,
|
|
"sampling/sampling_logp_difference/max": 0.7309276461601257,
|
|
"sampling/importance_sampling_ratio/min": 0.14433367550373077,
|
|
"sampling/importance_sampling_ratio/mean": 0.7738677859306335,
|
|
"sampling/importance_sampling_ratio/max": 1.5610729455947876,
|
|
"kl": 0.003069146640996223,
|
|
"entropy": 0.07247332442784682,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.001561623066664,
|
|
"epoch": 0.003203125,
|
|
"step": 123
|
|
},
|
|
{
|
|
"loss": -0.01703105866909027,
|
|
"grad_norm": 3.431641101837158,
|
|
"learning_rate": 3.051724137931034e-07,
|
|
"num_tokens": 1324326.0,
|
|
"completions/mean_length": 584.5,
|
|
"completions/min_length": 142.0,
|
|
"completions/max_length": 963.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 245.25,
|
|
"completions/min_terminated_length": 142.0,
|
|
"completions/max_terminated_length": 385.0,
|
|
"tools/call_frequency": 13.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.11874999105930328,
|
|
"rewards/reward_func/std": 0.22942085564136505,
|
|
"reward": 0.11874999105930328,
|
|
"reward_std": 0.22942085564136505,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0028252284973859787,
|
|
"sampling/sampling_logp_difference/max": 0.5441856384277344,
|
|
"sampling/importance_sampling_ratio/min": 0.3031948506832123,
|
|
"sampling/importance_sampling_ratio/mean": 0.9025987386703491,
|
|
"sampling/importance_sampling_ratio/max": 1.313743233680725,
|
|
"kl": 0.0011481192723294953,
|
|
"entropy": 0.05883795244153589,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.777608145028353,
|
|
"epoch": 0.0032291666666666666,
|
|
"step": 124
|
|
},
|
|
{
|
|
"loss": -0.30280601978302,
|
|
"grad_norm": 2.6027257442474365,
|
|
"learning_rate": 3.0344827586206897e-07,
|
|
"num_tokens": 1335830.0,
|
|
"completions/mean_length": 753.0,
|
|
"completions/min_length": 216.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 579.5,
|
|
"completions/min_terminated_length": 216.0,
|
|
"completions/max_terminated_length": 936.0,
|
|
"tools/call_frequency": 17.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.012500000186264515,
|
|
"rewards/reward_func/std": 0.06943651288747787,
|
|
"reward": 0.012500000186264515,
|
|
"reward_std": 0.06943650543689728,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018765154527500272,
|
|
"sampling/sampling_logp_difference/max": 0.9259822368621826,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8965011835098267,
|
|
"sampling/importance_sampling_ratio/max": 1.7344770431518555,
|
|
"kl": 0.00030621999849245185,
|
|
"entropy": 0.027476198854856193,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.867541201412678,
|
|
"epoch": 0.0032552083333333335,
|
|
"step": 125
|
|
},
|
|
{
|
|
"loss": 0.10897127538919449,
|
|
"grad_norm": 1.9802749156951904,
|
|
"learning_rate": 3.017241379310345e-07,
|
|
"num_tokens": 1346448.0,
|
|
"completions/mean_length": 641.75,
|
|
"completions/min_length": 168.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 364.25,
|
|
"completions/min_terminated_length": 168.0,
|
|
"completions/max_terminated_length": 899.0,
|
|
"tools/call_frequency": 15.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10104166716337204,
|
|
"rewards/reward_func/std": 0.2740944027900696,
|
|
"reward": 0.10104166716337204,
|
|
"reward_std": 0.2740943729877472,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001652171486057341,
|
|
"sampling/sampling_logp_difference/max": 0.35856008529663086,
|
|
"sampling/importance_sampling_ratio/min": 0.45409226417541504,
|
|
"sampling/importance_sampling_ratio/mean": 0.9129316806793213,
|
|
"sampling/importance_sampling_ratio/max": 1.6871345043182373,
|
|
"kl": 0.0017247018340640352,
|
|
"entropy": 0.03176611475646496,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.18392438441515,
|
|
"epoch": 0.00328125,
|
|
"step": 126
|
|
},
|
|
{
|
|
"loss": 0.6146202683448792,
|
|
"grad_norm": 2.7258055210113525,
|
|
"learning_rate": 3e-07,
|
|
"num_tokens": 1357953.0,
|
|
"completions/mean_length": 751.5,
|
|
"completions/min_length": 213.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 640.4000244140625,
|
|
"completions/min_terminated_length": 213.0,
|
|
"completions/max_terminated_length": 928.0,
|
|
"tools/call_frequency": 17.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3272569477558136,
|
|
"rewards/reward_func/std": 0.314951092004776,
|
|
"reward": 0.3272569477558136,
|
|
"reward_std": 0.314951092004776,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001846234081313014,
|
|
"sampling/sampling_logp_difference/max": 0.6850161552429199,
|
|
"sampling/importance_sampling_ratio/min": 0.47912609577178955,
|
|
"sampling/importance_sampling_ratio/mean": 1.1883745193481445,
|
|
"sampling/importance_sampling_ratio/max": 2.959009885787964,
|
|
"kl": 0.0011880160718646948,
|
|
"entropy": 0.03326657263096422,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.072050904855132,
|
|
"epoch": 0.0033072916666666667,
|
|
"step": 127
|
|
},
|
|
{
|
|
"loss": 0.12591040134429932,
|
|
"grad_norm": 2.7488999366760254,
|
|
"learning_rate": 2.982758620689655e-07,
|
|
"num_tokens": 1368101.0,
|
|
"completions/mean_length": 584.5,
|
|
"completions/min_length": 217.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 387.20001220703125,
|
|
"completions/min_terminated_length": 217.0,
|
|
"completions/max_terminated_length": 930.0,
|
|
"tools/call_frequency": 14.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.12395833432674408,
|
|
"rewards/reward_func/std": 0.25108641386032104,
|
|
"reward": 0.12395833432674408,
|
|
"reward_std": 0.25108641386032104,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0028649826999753714,
|
|
"sampling/sampling_logp_difference/max": 0.8482773303985596,
|
|
"sampling/importance_sampling_ratio/min": 0.2513328492641449,
|
|
"sampling/importance_sampling_ratio/mean": 0.8768019080162048,
|
|
"sampling/importance_sampling_ratio/max": 2.0821692943573,
|
|
"kl": 0.0012564498420033487,
|
|
"entropy": 0.041371886152774096,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.838409066200256,
|
|
"epoch": 0.0033333333333333335,
|
|
"step": 128
|
|
},
|
|
{
|
|
"loss": 0.03913319855928421,
|
|
"grad_norm": 2.6707587242126465,
|
|
"learning_rate": 2.96551724137931e-07,
|
|
"num_tokens": 1379498.0,
|
|
"completions/mean_length": 739.125,
|
|
"completions/min_length": 145.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 569.25,
|
|
"completions/min_terminated_length": 145.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 17.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06875000149011612,
|
|
"rewards/reward_func/std": 0.12517844140529633,
|
|
"reward": 0.06875000149011612,
|
|
"reward_std": 0.12517844140529633,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001434864941984415,
|
|
"sampling/sampling_logp_difference/max": 0.5436043739318848,
|
|
"sampling/importance_sampling_ratio/min": 0.32730746269226074,
|
|
"sampling/importance_sampling_ratio/mean": 1.0741466283798218,
|
|
"sampling/importance_sampling_ratio/max": 1.979141354560852,
|
|
"kl": 0.0005071343352938129,
|
|
"entropy": 0.01955541208735667,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.52513245306909,
|
|
"epoch": 0.003359375,
|
|
"step": 129
|
|
},
|
|
{
|
|
"loss": 0.08846982568502426,
|
|
"grad_norm": 1.8919341564178467,
|
|
"learning_rate": 2.948275862068965e-07,
|
|
"num_tokens": 1390101.0,
|
|
"completions/mean_length": 642.125,
|
|
"completions/min_length": 172.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 491.0,
|
|
"completions/min_terminated_length": 172.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 16.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.08645832538604736,
|
|
"rewards/reward_func/std": 0.23434887826442719,
|
|
"reward": 0.08645832538604736,
|
|
"reward_std": 0.234348863363266,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016239171382039785,
|
|
"sampling/sampling_logp_difference/max": 0.5126781463623047,
|
|
"sampling/importance_sampling_ratio/min": 0.6199666261672974,
|
|
"sampling/importance_sampling_ratio/mean": 0.7938264608383179,
|
|
"sampling/importance_sampling_ratio/max": 1.23106849193573,
|
|
"kl": 0.0021221743863861775,
|
|
"entropy": 0.031170871574431658,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.094140378758311,
|
|
"epoch": 0.0033854166666666668,
|
|
"step": 130
|
|
},
|
|
{
|
|
"loss": -0.16358976066112518,
|
|
"grad_norm": 2.973278045654297,
|
|
"learning_rate": 2.93103448275862e-07,
|
|
"num_tokens": 1399065.0,
|
|
"completions/mean_length": 435.5,
|
|
"completions/min_length": 142.0,
|
|
"completions/max_length": 908.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/mean_terminated_length": 435.5,
|
|
"completions/min_terminated_length": 142.0,
|
|
"completions/max_terminated_length": 908.0,
|
|
"tools/call_frequency": 10.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2541666626930237,
|
|
"rewards/reward_func/std": 0.3920527398586273,
|
|
"reward": 0.2541666626930237,
|
|
"reward_std": 0.3920527398586273,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.005152893718332052,
|
|
"sampling/sampling_logp_difference/max": 0.6487990617752075,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.6446778774261475,
|
|
"sampling/importance_sampling_ratio/max": 1.1622810363769531,
|
|
"kl": 0.0012626445568457711,
|
|
"entropy": 0.07517135958187282,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.47301589883864,
|
|
"epoch": 0.003411458333333333,
|
|
"step": 131
|
|
},
|
|
{
|
|
"loss": 0.044558919966220856,
|
|
"grad_norm": 3.140397787094116,
|
|
"learning_rate": 2.913793103448276e-07,
|
|
"num_tokens": 1407431.0,
|
|
"completions/mean_length": 361.0,
|
|
"completions/min_length": 142.0,
|
|
"completions/max_length": 897.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 182.83334350585938,
|
|
"completions/min_terminated_length": 142.0,
|
|
"completions/max_terminated_length": 256.0,
|
|
"tools/call_frequency": 9.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.11666667461395264,
|
|
"rewards/reward_func/std": 0.3244348168373108,
|
|
"reward": 0.11666667461395264,
|
|
"reward_std": 0.3244347870349884,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002882197964936495,
|
|
"sampling/sampling_logp_difference/max": 0.6759878396987915,
|
|
"sampling/importance_sampling_ratio/min": 0.5223076939582825,
|
|
"sampling/importance_sampling_ratio/mean": 0.9067434072494507,
|
|
"sampling/importance_sampling_ratio/max": 1.3223317861557007,
|
|
"kl": 0.0009833206740950118,
|
|
"entropy": 0.042295850231312215,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.757221641018987,
|
|
"epoch": 0.0034375,
|
|
"step": 132
|
|
},
|
|
{
|
|
"loss": 0.24208419024944305,
|
|
"grad_norm": 2.56244158744812,
|
|
"learning_rate": 2.896551724137931e-07,
|
|
"num_tokens": 1418078.0,
|
|
"completions/mean_length": 644.5,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 369.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 900.0,
|
|
"tools/call_frequency": 14.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.22291666269302368,
|
|
"rewards/reward_func/std": 0.37381428480148315,
|
|
"reward": 0.22291666269302368,
|
|
"reward_std": 0.37381428480148315,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020186526235193014,
|
|
"sampling/sampling_logp_difference/max": 0.6855125427246094,
|
|
"sampling/importance_sampling_ratio/min": 0.3692649006843567,
|
|
"sampling/importance_sampling_ratio/mean": 1.1208091974258423,
|
|
"sampling/importance_sampling_ratio/max": 2.07541823387146,
|
|
"kl": 0.001040317465140106,
|
|
"entropy": 0.03527856047730893,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.384221917018294,
|
|
"epoch": 0.003463541666666667,
|
|
"step": 133
|
|
},
|
|
{
|
|
"loss": 0.09889425337314606,
|
|
"grad_norm": 1.683709740638733,
|
|
"learning_rate": 2.8793103448275865e-07,
|
|
"num_tokens": 1427936.0,
|
|
"completions/mean_length": 547.625,
|
|
"completions/min_length": 144.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 443.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"completions/max_terminated_length": 954.0,
|
|
"tools/call_frequency": 14.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.0617559514939785,
|
|
"rewards/reward_func/std": 0.22752776741981506,
|
|
"reward": 0.0617559514939785,
|
|
"reward_std": 0.22752776741981506,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0025728922337293625,
|
|
"sampling/sampling_logp_difference/max": 0.5126767158508301,
|
|
"sampling/importance_sampling_ratio/min": 0.22409002482891083,
|
|
"sampling/importance_sampling_ratio/mean": 0.8655706644058228,
|
|
"sampling/importance_sampling_ratio/max": 1.5354048013687134,
|
|
"kl": 0.0012264249126019422,
|
|
"entropy": 0.039603013661690056,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.23488742671907,
|
|
"epoch": 0.0034895833333333333,
|
|
"step": 134
|
|
},
|
|
{
|
|
"loss": 0.27377450466156006,
|
|
"grad_norm": 13.312023162841797,
|
|
"learning_rate": 2.8620689655172416e-07,
|
|
"num_tokens": 1438758.0,
|
|
"completions/mean_length": 668.875,
|
|
"completions/min_length": 231.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 415.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"completions/max_terminated_length": 902.0,
|
|
"tools/call_frequency": 16.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18437501788139343,
|
|
"rewards/reward_func/std": 0.22067886590957642,
|
|
"reward": 0.18437501788139343,
|
|
"reward_std": 0.22067886590957642,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002425384009256959,
|
|
"sampling/sampling_logp_difference/max": 2.337886333465576,
|
|
"sampling/importance_sampling_ratio/min": 0.03648517653346062,
|
|
"sampling/importance_sampling_ratio/mean": 0.6627132296562195,
|
|
"sampling/importance_sampling_ratio/max": 1.0501126050949097,
|
|
"kl": 0.002540460472118866,
|
|
"entropy": 0.042934882279951125,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.611063748598099,
|
|
"epoch": 0.003515625,
|
|
"step": 135
|
|
},
|
|
{
|
|
"loss": -0.1222410649061203,
|
|
"grad_norm": 3.2033724784851074,
|
|
"learning_rate": 2.8448275862068967e-07,
|
|
"num_tokens": 1448026.0,
|
|
"completions/mean_length": 474.0,
|
|
"completions/min_length": 168.0,
|
|
"completions/max_length": 898.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 413.4285888671875,
|
|
"completions/min_terminated_length": 168.0,
|
|
"completions/max_terminated_length": 896.0,
|
|
"tools/call_frequency": 12.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18246527016162872,
|
|
"rewards/reward_func/std": 0.31536296010017395,
|
|
"reward": 0.18246527016162872,
|
|
"reward_std": 0.31536293029785156,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003783475374802947,
|
|
"sampling/sampling_logp_difference/max": 1.0369452238082886,
|
|
"sampling/importance_sampling_ratio/min": 0.32026243209838867,
|
|
"sampling/importance_sampling_ratio/mean": 0.7692611217498779,
|
|
"sampling/importance_sampling_ratio/max": 1.2486343383789062,
|
|
"kl": 0.0027144144305566442,
|
|
"entropy": 0.06231480755377561,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.12981073744595,
|
|
"epoch": 0.0035416666666666665,
|
|
"step": 136
|
|
},
|
|
{
|
|
"loss": 0.1233871579170227,
|
|
"grad_norm": 2.585420608520508,
|
|
"learning_rate": 2.827586206896552e-07,
|
|
"num_tokens": 1459005.0,
|
|
"completions/mean_length": 686.125,
|
|
"completions/min_length": 266.0,
|
|
"completions/max_length": 956.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 529.6000366210938,
|
|
"completions/min_terminated_length": 266.0,
|
|
"completions/max_terminated_length": 906.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.19687500596046448,
|
|
"rewards/reward_func/std": 0.18741069734096527,
|
|
"reward": 0.19687500596046448,
|
|
"reward_std": 0.18741069734096527,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019413406262174249,
|
|
"sampling/sampling_logp_difference/max": 0.6200988292694092,
|
|
"sampling/importance_sampling_ratio/min": 0.35567551851272583,
|
|
"sampling/importance_sampling_ratio/mean": 0.9653811454772949,
|
|
"sampling/importance_sampling_ratio/max": 1.7431609630584717,
|
|
"kl": 0.0010818231569373893,
|
|
"entropy": 0.035575162852182984,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.70743653923273,
|
|
"epoch": 0.0035677083333333333,
|
|
"step": 137
|
|
},
|
|
{
|
|
"loss": -0.020581696182489395,
|
|
"grad_norm": 1.2376840114593506,
|
|
"learning_rate": 2.810344827586207e-07,
|
|
"num_tokens": 1469879.0,
|
|
"completions/mean_length": 674.375,
|
|
"completions/min_length": 168.0,
|
|
"completions/max_length": 965.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 418.25,
|
|
"completions/min_terminated_length": 168.0,
|
|
"completions/max_terminated_length": 907.0,
|
|
"tools/call_frequency": 15.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05000000074505806,
|
|
"rewards/reward_func/std": 0.08017837256193161,
|
|
"reward": 0.05000000074505806,
|
|
"reward_std": 0.08017837256193161,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019047798123210669,
|
|
"sampling/sampling_logp_difference/max": 0.4973316192626953,
|
|
"sampling/importance_sampling_ratio/min": 0.32398220896720886,
|
|
"sampling/importance_sampling_ratio/mean": 0.6056469082832336,
|
|
"sampling/importance_sampling_ratio/max": 0.8678992986679077,
|
|
"kl": 0.0003474869804449554,
|
|
"entropy": 0.032995211658999324,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.432136783376336,
|
|
"epoch": 0.00359375,
|
|
"step": 138
|
|
},
|
|
{
|
|
"loss": 0.08556556701660156,
|
|
"grad_norm": 6.4354047775268555,
|
|
"learning_rate": 2.793103448275862e-07,
|
|
"num_tokens": 1478551.0,
|
|
"completions/mean_length": 399.625,
|
|
"completions/min_length": 177.0,
|
|
"completions/max_length": 901.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 328.0,
|
|
"completions/min_terminated_length": 177.0,
|
|
"completions/max_terminated_length": 888.0,
|
|
"tools/call_frequency": 10.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.11874999850988388,
|
|
"rewards/reward_func/std": 0.3326704800128937,
|
|
"reward": 0.11874999850988388,
|
|
"reward_std": 0.3326704502105713,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003122760448604822,
|
|
"sampling/sampling_logp_difference/max": 0.2986990809440613,
|
|
"sampling/importance_sampling_ratio/min": 0.5903478860855103,
|
|
"sampling/importance_sampling_ratio/mean": 1.046547293663025,
|
|
"sampling/importance_sampling_ratio/max": 1.5082789659500122,
|
|
"kl": 0.003801928522079834,
|
|
"entropy": 0.05084260122384876,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.267111564055085,
|
|
"epoch": 0.0036197916666666666,
|
|
"step": 139
|
|
},
|
|
{
|
|
"loss": -0.11556712538003922,
|
|
"grad_norm": 3.73350191116333,
|
|
"learning_rate": 2.775862068965517e-07,
|
|
"num_tokens": 1490065.0,
|
|
"completions/mean_length": 753.5,
|
|
"completions/min_length": 196.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 699.6666870117188,
|
|
"completions/min_terminated_length": 196.0,
|
|
"completions/max_terminated_length": 948.0,
|
|
"tools/call_frequency": 17.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05000000074505806,
|
|
"rewards/reward_func/std": 0.08017837256193161,
|
|
"reward": 0.05000000074505806,
|
|
"reward_std": 0.08017837256193161,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014235425041988492,
|
|
"sampling/sampling_logp_difference/max": 0.9361461400985718,
|
|
"sampling/importance_sampling_ratio/min": 0.6736374497413635,
|
|
"sampling/importance_sampling_ratio/mean": 1.4344265460968018,
|
|
"sampling/importance_sampling_ratio/max": 2.37829852104187,
|
|
"kl": 0.0003969354797845881,
|
|
"entropy": 0.03235747164580971,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.2500683311373,
|
|
"epoch": 0.0036458333333333334,
|
|
"step": 140
|
|
},
|
|
{
|
|
"loss": 0.006581395864486694,
|
|
"grad_norm": 2.584792375564575,
|
|
"learning_rate": 2.758620689655172e-07,
|
|
"num_tokens": 1500634.0,
|
|
"completions/mean_length": 637.25,
|
|
"completions/min_length": 184.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 567.6666870117188,
|
|
"completions/min_terminated_length": 184.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 16.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2881944477558136,
|
|
"rewards/reward_func/std": 0.3029133677482605,
|
|
"reward": 0.2881944477558136,
|
|
"reward_std": 0.3029133677482605,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014153514057397842,
|
|
"sampling/sampling_logp_difference/max": 0.3808797597885132,
|
|
"sampling/importance_sampling_ratio/min": 0.6545470952987671,
|
|
"sampling/importance_sampling_ratio/mean": 1.0732766389846802,
|
|
"sampling/importance_sampling_ratio/max": 2.147284984588623,
|
|
"kl": 0.0011165879823238356,
|
|
"entropy": 0.03501797135686502,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.01376754231751,
|
|
"epoch": 0.003671875,
|
|
"step": 141
|
|
},
|
|
{
|
|
"loss": 0.21006156504154205,
|
|
"grad_norm": 2.0534121990203857,
|
|
"learning_rate": 2.741379310344827e-07,
|
|
"num_tokens": 1511323.0,
|
|
"completions/mean_length": 651.25,
|
|
"completions/min_length": 198.0,
|
|
"completions/max_length": 934.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 378.75,
|
|
"completions/min_terminated_length": 198.0,
|
|
"completions/max_terminated_length": 880.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17812499403953552,
|
|
"rewards/reward_func/std": 0.29744070768356323,
|
|
"reward": 0.17812499403953552,
|
|
"reward_std": 0.29744070768356323,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0024400060065090656,
|
|
"sampling/sampling_logp_difference/max": 0.937833309173584,
|
|
"sampling/importance_sampling_ratio/min": 0.3932560682296753,
|
|
"sampling/importance_sampling_ratio/mean": 0.8978743553161621,
|
|
"sampling/importance_sampling_ratio/max": 1.6709332466125488,
|
|
"kl": 0.0007719381819697446,
|
|
"entropy": 0.037543563172221184,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.668473599478602,
|
|
"epoch": 0.0036979166666666666,
|
|
"step": 142
|
|
},
|
|
{
|
|
"loss": 0.22358570992946625,
|
|
"grad_norm": 2.802809238433838,
|
|
"learning_rate": 2.724137931034483e-07,
|
|
"num_tokens": 1521566.0,
|
|
"completions/mean_length": 593.875,
|
|
"completions/min_length": 232.0,
|
|
"completions/max_length": 951.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 252.25,
|
|
"completions/min_terminated_length": 232.0,
|
|
"completions/max_terminated_length": 298.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2697916626930237,
|
|
"rewards/reward_func/std": 0.35118669271469116,
|
|
"reward": 0.2697916626930237,
|
|
"reward_std": 0.35118669271469116,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022893566638231277,
|
|
"sampling/sampling_logp_difference/max": 0.49288177490234375,
|
|
"sampling/importance_sampling_ratio/min": 0.3620584309101105,
|
|
"sampling/importance_sampling_ratio/mean": 1.072188377380371,
|
|
"sampling/importance_sampling_ratio/max": 1.9088119268417358,
|
|
"kl": 0.0011805686393699943,
|
|
"entropy": 0.04600911698071286,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.665387984365225,
|
|
"epoch": 0.0037239583333333335,
|
|
"step": 143
|
|
},
|
|
{
|
|
"loss": 0.2978719174861908,
|
|
"grad_norm": 6.367245197296143,
|
|
"learning_rate": 2.706896551724138e-07,
|
|
"num_tokens": 1531557.0,
|
|
"completions/mean_length": 564.125,
|
|
"completions/min_length": 150.0,
|
|
"completions/max_length": 949.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 229.25,
|
|
"completions/min_terminated_length": 150.0,
|
|
"completions/max_terminated_length": 303.0,
|
|
"tools/call_frequency": 13.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3103422522544861,
|
|
"rewards/reward_func/std": 0.350017786026001,
|
|
"reward": 0.3103422522544861,
|
|
"reward_std": 0.350017786026001,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022420454770326614,
|
|
"sampling/sampling_logp_difference/max": 0.8773374557495117,
|
|
"sampling/importance_sampling_ratio/min": 0.6984423995018005,
|
|
"sampling/importance_sampling_ratio/mean": 1.1002775430679321,
|
|
"sampling/importance_sampling_ratio/max": 2.358316421508789,
|
|
"kl": 0.0016524331467735465,
|
|
"entropy": 0.04010078724240884,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.34181209281087,
|
|
"epoch": 0.00375,
|
|
"step": 144
|
|
},
|
|
{
|
|
"loss": -0.2550424039363861,
|
|
"grad_norm": 1.3035966157913208,
|
|
"learning_rate": 2.689655172413793e-07,
|
|
"num_tokens": 1542951.0,
|
|
"completions/mean_length": 738.625,
|
|
"completions/min_length": 152.0,
|
|
"completions/max_length": 955.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 560.0,
|
|
"completions/min_terminated_length": 152.0,
|
|
"completions/max_terminated_length": 946.0,
|
|
"tools/call_frequency": 17.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03593749925494194,
|
|
"rewards/reward_func/std": 0.1059475690126419,
|
|
"reward": 0.03593749925494194,
|
|
"reward_std": 0.1059475764632225,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014282736228778958,
|
|
"sampling/sampling_logp_difference/max": 0.45047712326049805,
|
|
"sampling/importance_sampling_ratio/min": 0.42352357506752014,
|
|
"sampling/importance_sampling_ratio/mean": 0.9527370929718018,
|
|
"sampling/importance_sampling_ratio/max": 1.4430091381072998,
|
|
"kl": 0.00044436611472065124,
|
|
"entropy": 0.022874564107041806,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.719373594969511,
|
|
"epoch": 0.0037760416666666667,
|
|
"step": 145
|
|
},
|
|
{
|
|
"loss": 0.20484033226966858,
|
|
"grad_norm": 1.5121691226959229,
|
|
"learning_rate": 2.672413793103448e-07,
|
|
"num_tokens": 1554221.0,
|
|
"completions/mean_length": 724.875,
|
|
"completions/min_length": 193.0,
|
|
"completions/max_length": 1015.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 631.0,
|
|
"completions/min_terminated_length": 193.0,
|
|
"completions/max_terminated_length": 1015.0,
|
|
"tools/call_frequency": 17.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.24464285373687744,
|
|
"rewards/reward_func/std": 0.4099794030189514,
|
|
"reward": 0.24464285373687744,
|
|
"reward_std": 0.40997934341430664,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017402005614712834,
|
|
"sampling/sampling_logp_difference/max": 0.5598816871643066,
|
|
"sampling/importance_sampling_ratio/min": 0.33270174264907837,
|
|
"sampling/importance_sampling_ratio/mean": 0.8581236600875854,
|
|
"sampling/importance_sampling_ratio/max": 1.1936233043670654,
|
|
"kl": 0.0015142025713430485,
|
|
"entropy": 0.03032794565660879,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.413477493450046,
|
|
"epoch": 0.0038020833333333335,
|
|
"step": 146
|
|
},
|
|
{
|
|
"loss": 0.2457561045885086,
|
|
"grad_norm": 1.320375919342041,
|
|
"learning_rate": 2.655172413793103e-07,
|
|
"num_tokens": 1564989.0,
|
|
"completions/mean_length": 661.125,
|
|
"completions/min_length": 180.0,
|
|
"completions/max_length": 980.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 383.25,
|
|
"completions/min_terminated_length": 180.0,
|
|
"completions/max_terminated_length": 894.0,
|
|
"tools/call_frequency": 15.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.20208334922790527,
|
|
"rewards/reward_func/std": 0.3319540023803711,
|
|
"reward": 0.20208334922790527,
|
|
"reward_std": 0.3319540023803711,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001671720645390451,
|
|
"sampling/sampling_logp_difference/max": 0.38815009593963623,
|
|
"sampling/importance_sampling_ratio/min": 0.5895294547080994,
|
|
"sampling/importance_sampling_ratio/mean": 0.8562068939208984,
|
|
"sampling/importance_sampling_ratio/max": 1.3771353960037231,
|
|
"kl": 0.0014014716437031893,
|
|
"entropy": 0.03570698096882552,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.84879719465971,
|
|
"epoch": 0.003828125,
|
|
"step": 147
|
|
},
|
|
{
|
|
"loss": 0.0003149906697217375,
|
|
"grad_norm": 1.583635687828064,
|
|
"learning_rate": 2.6379310344827587e-07,
|
|
"num_tokens": 1577829.0,
|
|
"completions/mean_length": 919.625,
|
|
"completions/min_length": 865.0,
|
|
"completions/max_length": 978.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 930.5,
|
|
"completions/min_terminated_length": 928.0,
|
|
"completions/max_terminated_length": 933.0,
|
|
"tools/call_frequency": 21.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05000000074505806,
|
|
"rewards/reward_func/std": 0.0,
|
|
"reward": 0.05000000074505806,
|
|
"reward_std": 0.0,
|
|
"frac_reward_zero_std": 1.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001288810046389699,
|
|
"sampling/sampling_logp_difference/max": 0.7733561992645264,
|
|
"sampling/importance_sampling_ratio/min": 0.3952680230140686,
|
|
"sampling/importance_sampling_ratio/mean": 0.8995804786682129,
|
|
"sampling/importance_sampling_ratio/max": 1.4702492952346802,
|
|
"kl": 0.0003263111190676682,
|
|
"entropy": 0.01311570534016937,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.769840428605676,
|
|
"epoch": 0.0038541666666666668,
|
|
"step": 148
|
|
},
|
|
{
|
|
"loss": -0.17162922024726868,
|
|
"grad_norm": 2.691833734512329,
|
|
"learning_rate": 2.620689655172414e-07,
|
|
"num_tokens": 1589270.0,
|
|
"completions/mean_length": 745.5,
|
|
"completions/min_length": 185.0,
|
|
"completions/max_length": 918.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 248.5,
|
|
"completions/min_terminated_length": 185.0,
|
|
"completions/max_terminated_length": 312.0,
|
|
"tools/call_frequency": 18.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13750000298023224,
|
|
"rewards/reward_func/std": 0.21557646989822388,
|
|
"reward": 0.13750000298023224,
|
|
"reward_std": 0.21557646989822388,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0011822429951280355,
|
|
"sampling/sampling_logp_difference/max": 0.283680260181427,
|
|
"sampling/importance_sampling_ratio/min": 0.42503345012664795,
|
|
"sampling/importance_sampling_ratio/mean": 0.9086511135101318,
|
|
"sampling/importance_sampling_ratio/max": 1.7465705871582031,
|
|
"kl": 0.0006872454273434414,
|
|
"entropy": 0.02307722863042727,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.169485842809081,
|
|
"epoch": 0.003880208333333333,
|
|
"step": 149
|
|
},
|
|
{
|
|
"loss": 0.2788282632827759,
|
|
"grad_norm": 1.9461345672607422,
|
|
"learning_rate": 2.603448275862069e-07,
|
|
"num_tokens": 1600047.0,
|
|
"completions/mean_length": 662.5,
|
|
"completions/min_length": 227.0,
|
|
"completions/max_length": 922.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 415.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"completions/max_terminated_length": 916.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.36250001192092896,
|
|
"rewards/reward_func/std": 0.30039656162261963,
|
|
"reward": 0.36250001192092896,
|
|
"reward_std": 0.30039656162261963,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018959178123623133,
|
|
"sampling/sampling_logp_difference/max": 0.5641241073608398,
|
|
"sampling/importance_sampling_ratio/min": 0.4278271794319153,
|
|
"sampling/importance_sampling_ratio/mean": 1.0406675338745117,
|
|
"sampling/importance_sampling_ratio/max": 1.5236719846725464,
|
|
"kl": 0.0020035318739246577,
|
|
"entropy": 0.038233465573284775,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.961871843785048,
|
|
"epoch": 0.00390625,
|
|
"step": 150
|
|
},
|
|
{
|
|
"loss": 0.11574936658143997,
|
|
"grad_norm": 1.5744644403457642,
|
|
"learning_rate": 2.586206896551724e-07,
|
|
"num_tokens": 1611415.0,
|
|
"completions/mean_length": 735.875,
|
|
"completions/min_length": 197.0,
|
|
"completions/max_length": 939.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 617.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"completions/max_terminated_length": 919.0,
|
|
"tools/call_frequency": 17.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16041666269302368,
|
|
"rewards/reward_func/std": 0.27108508348464966,
|
|
"reward": 0.16041666269302368,
|
|
"reward_std": 0.27108505368232727,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001662094728089869,
|
|
"sampling/sampling_logp_difference/max": 0.8482754230499268,
|
|
"sampling/importance_sampling_ratio/min": 0.41452887654304504,
|
|
"sampling/importance_sampling_ratio/mean": 0.8144710659980774,
|
|
"sampling/importance_sampling_ratio/max": 1.0838565826416016,
|
|
"kl": 0.000747345334275451,
|
|
"entropy": 0.025778961018659174,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.242910644039512,
|
|
"epoch": 0.003932291666666666,
|
|
"step": 151
|
|
},
|
|
{
|
|
"loss": 0.24026885628700256,
|
|
"grad_norm": 1.6332927942276,
|
|
"learning_rate": 2.5689655172413796e-07,
|
|
"num_tokens": 1621503.0,
|
|
"completions/mean_length": 575.25,
|
|
"completions/min_length": 149.0,
|
|
"completions/max_length": 934.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 381.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"completions/max_terminated_length": 934.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21857638657093048,
|
|
"rewards/reward_func/std": 0.3464857041835785,
|
|
"reward": 0.21857638657093048,
|
|
"reward_std": 0.3464856743812561,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0023415915202349424,
|
|
"sampling/sampling_logp_difference/max": 0.5268945693969727,
|
|
"sampling/importance_sampling_ratio/min": 0.25896450877189636,
|
|
"sampling/importance_sampling_ratio/mean": 0.8156750202178955,
|
|
"sampling/importance_sampling_ratio/max": 2.1791300773620605,
|
|
"kl": 0.0008446880115116073,
|
|
"entropy": 0.044077472877688706,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.880278185009956,
|
|
"epoch": 0.003958333333333334,
|
|
"step": 152
|
|
},
|
|
{
|
|
"loss": -0.23129448294639587,
|
|
"grad_norm": 3.1814544200897217,
|
|
"learning_rate": 2.5517241379310346e-07,
|
|
"num_tokens": 1633652.0,
|
|
"completions/mean_length": 834.0,
|
|
"completions/min_length": 261.0,
|
|
"completions/max_length": 947.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 589.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"completions/max_terminated_length": 917.0,
|
|
"tools/call_frequency": 19.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0008801064686849713,
|
|
"sampling/sampling_logp_difference/max": 1.1254721879959106,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.9567307233810425,
|
|
"sampling/importance_sampling_ratio/max": 1.3288164138793945,
|
|
"kl": 0.0004218181219357575,
|
|
"entropy": 0.017437056871131063,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.66511089541018,
|
|
"epoch": 0.003984375,
|
|
"step": 153
|
|
},
|
|
{
|
|
"loss": 0.2771526575088501,
|
|
"grad_norm": 1.4955064058303833,
|
|
"learning_rate": 2.5344827586206897e-07,
|
|
"num_tokens": 1644354.0,
|
|
"completions/mean_length": 653.125,
|
|
"completions/min_length": 167.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 203.6666717529297,
|
|
"completions/min_terminated_length": 167.0,
|
|
"completions/max_terminated_length": 223.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.14652776718139648,
|
|
"rewards/reward_func/std": 0.2309771031141281,
|
|
"reward": 0.14652776718139648,
|
|
"reward_std": 0.23097708821296692,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0011717748129740357,
|
|
"sampling/sampling_logp_difference/max": 0.2497415542602539,
|
|
"sampling/importance_sampling_ratio/min": 0.4442828893661499,
|
|
"sampling/importance_sampling_ratio/mean": 0.8846491575241089,
|
|
"sampling/importance_sampling_ratio/max": 1.1166706085205078,
|
|
"kl": 0.0018671902912501537,
|
|
"entropy": 0.024821540981065482,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.410500617697835,
|
|
"epoch": 0.0040104166666666665,
|
|
"step": 154
|
|
},
|
|
{
|
|
"loss": 0.0968727096915245,
|
|
"grad_norm": 1.5175065994262695,
|
|
"learning_rate": 2.517241379310345e-07,
|
|
"num_tokens": 1655211.0,
|
|
"completions/mean_length": 672.25,
|
|
"completions/min_length": 205.0,
|
|
"completions/max_length": 946.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 529.2000122070312,
|
|
"completions/min_terminated_length": 205.0,
|
|
"completions/max_terminated_length": 946.0,
|
|
"tools/call_frequency": 15.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23645833134651184,
|
|
"rewards/reward_func/std": 0.28497207164764404,
|
|
"reward": 0.23645833134651184,
|
|
"reward_std": 0.28497204184532166,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016737524420022964,
|
|
"sampling/sampling_logp_difference/max": 0.3886861801147461,
|
|
"sampling/importance_sampling_ratio/min": 0.39735281467437744,
|
|
"sampling/importance_sampling_ratio/mean": 0.8298213481903076,
|
|
"sampling/importance_sampling_ratio/max": 1.4523310661315918,
|
|
"kl": 0.0007411708174913656,
|
|
"entropy": 0.031021748669445515,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.680708745494485,
|
|
"epoch": 0.004036458333333334,
|
|
"step": 155
|
|
},
|
|
{
|
|
"loss": 0.024390675127506256,
|
|
"grad_norm": 1.9517337083816528,
|
|
"learning_rate": 2.5e-07,
|
|
"num_tokens": 1665168.0,
|
|
"completions/mean_length": 560.25,
|
|
"completions/min_length": 145.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 440.16668701171875,
|
|
"completions/min_terminated_length": 145.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 13.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.07309027761220932,
|
|
"rewards/reward_func/std": 0.1984548717737198,
|
|
"reward": 0.07309027761220932,
|
|
"reward_std": 0.1984548717737198,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019503082148730755,
|
|
"sampling/sampling_logp_difference/max": 0.5216224193572998,
|
|
"sampling/importance_sampling_ratio/min": 0.3797282874584198,
|
|
"sampling/importance_sampling_ratio/mean": 0.8693698644638062,
|
|
"sampling/importance_sampling_ratio/max": 1.4351552724838257,
|
|
"kl": 0.0011887399041370372,
|
|
"entropy": 0.03445904515683651,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.031802034005523,
|
|
"epoch": 0.0040625,
|
|
"step": 156
|
|
},
|
|
{
|
|
"loss": 0.37649089097976685,
|
|
"grad_norm": 1.654868721961975,
|
|
"learning_rate": 2.482758620689655e-07,
|
|
"num_tokens": 1675264.0,
|
|
"completions/mean_length": 577.125,
|
|
"completions/min_length": 144.0,
|
|
"completions/max_length": 922.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 377.8000183105469,
|
|
"completions/min_terminated_length": 144.0,
|
|
"completions/max_terminated_length": 915.0,
|
|
"tools/call_frequency": 14.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16398808360099792,
|
|
"rewards/reward_func/std": 0.31408387422561646,
|
|
"reward": 0.16398808360099792,
|
|
"reward_std": 0.31408387422561646,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020221041049808264,
|
|
"sampling/sampling_logp_difference/max": 0.4436972737312317,
|
|
"sampling/importance_sampling_ratio/min": 0.38680627942085266,
|
|
"sampling/importance_sampling_ratio/mean": 1.099252700805664,
|
|
"sampling/importance_sampling_ratio/max": 2.123635768890381,
|
|
"kl": 0.0011169990521011641,
|
|
"entropy": 0.042600430257152766,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.036935398355126,
|
|
"epoch": 0.0040885416666666665,
|
|
"step": 157
|
|
},
|
|
{
|
|
"loss": 0.10837496072053909,
|
|
"grad_norm": 1.4167053699493408,
|
|
"learning_rate": 2.46551724137931e-07,
|
|
"num_tokens": 1686182.0,
|
|
"completions/mean_length": 679.25,
|
|
"completions/min_length": 235.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 254.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"completions/max_terminated_length": 273.0,
|
|
"tools/call_frequency": 14.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.140625,
|
|
"rewards/reward_func/std": 0.23705315589904785,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.23705314099788666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002403603633865714,
|
|
"sampling/sampling_logp_difference/max": 0.49289655685424805,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.5721709132194519,
|
|
"sampling/importance_sampling_ratio/max": 0.9959339499473572,
|
|
"kl": 0.0010291908415638318,
|
|
"entropy": 0.04185775015503168,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.25677259452641,
|
|
"epoch": 0.004114583333333333,
|
|
"step": 158
|
|
},
|
|
{
|
|
"loss": -0.06798543781042099,
|
|
"grad_norm": 2.312790870666504,
|
|
"learning_rate": 2.448275862068965e-07,
|
|
"num_tokens": 1696296.0,
|
|
"completions/mean_length": 579.125,
|
|
"completions/min_length": 149.0,
|
|
"completions/max_length": 920.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 478.16668701171875,
|
|
"completions/min_terminated_length": 149.0,
|
|
"completions/max_terminated_length": 920.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3333333432674408,
|
|
"rewards/reward_func/std": 0.30220749974250793,
|
|
"reward": 0.3333333432674408,
|
|
"reward_std": 0.30220746994018555,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.00243158801458776,
|
|
"sampling/sampling_logp_difference/max": 0.4080771207809448,
|
|
"sampling/importance_sampling_ratio/min": 0.48819324374198914,
|
|
"sampling/importance_sampling_ratio/mean": 0.9108745455741882,
|
|
"sampling/importance_sampling_ratio/max": 1.325175404548645,
|
|
"kl": 0.0017493370396550745,
|
|
"entropy": 0.046818539733067155,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.987576158717275,
|
|
"epoch": 0.004140625,
|
|
"step": 159
|
|
},
|
|
{
|
|
"loss": 0.22556769847869873,
|
|
"grad_norm": 1.9716132879257202,
|
|
"learning_rate": 2.43103448275862e-07,
|
|
"num_tokens": 1706387.0,
|
|
"completions/mean_length": 577.25,
|
|
"completions/min_length": 199.0,
|
|
"completions/max_length": 960.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 235.75,
|
|
"completions/min_terminated_length": 199.0,
|
|
"completions/max_terminated_length": 265.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10312499850988388,
|
|
"rewards/reward_func/std": 0.3402828574180603,
|
|
"reward": 0.10312499850988388,
|
|
"reward_std": 0.3402828276157379,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0024672155268490314,
|
|
"sampling/sampling_logp_difference/max": 0.750032901763916,
|
|
"sampling/importance_sampling_ratio/min": 0.6053751111030579,
|
|
"sampling/importance_sampling_ratio/mean": 1.3247417211532593,
|
|
"sampling/importance_sampling_ratio/max": 2.3258752822875977,
|
|
"kl": 0.0008925600977818249,
|
|
"entropy": 0.044351928401738405,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.813231285661459,
|
|
"epoch": 0.004166666666666667,
|
|
"step": 160
|
|
},
|
|
{
|
|
"loss": 0.18063189089298248,
|
|
"grad_norm": 2.7704994678497314,
|
|
"learning_rate": 2.413793103448276e-07,
|
|
"num_tokens": 1716572.0,
|
|
"completions/mean_length": 589.125,
|
|
"completions/min_length": 193.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 395.3999938964844,
|
|
"completions/min_terminated_length": 193.0,
|
|
"completions/max_terminated_length": 919.0,
|
|
"tools/call_frequency": 14.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.28593751788139343,
|
|
"rewards/reward_func/std": 0.39638084173202515,
|
|
"reward": 0.28593751788139343,
|
|
"reward_std": 0.39638081192970276,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002102903090417385,
|
|
"sampling/sampling_logp_difference/max": 0.5274801850318909,
|
|
"sampling/importance_sampling_ratio/min": 0.3482665419578552,
|
|
"sampling/importance_sampling_ratio/mean": 0.9032816886901855,
|
|
"sampling/importance_sampling_ratio/max": 1.2330471277236938,
|
|
"kl": 0.0012305123559599451,
|
|
"entropy": 0.04367574956268072,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.391089528799057,
|
|
"epoch": 0.004192708333333333,
|
|
"step": 161
|
|
},
|
|
{
|
|
"loss": 0.2699795365333557,
|
|
"grad_norm": 3.951411485671997,
|
|
"learning_rate": 2.396551724137931e-07,
|
|
"num_tokens": 1727311.0,
|
|
"completions/mean_length": 657.375,
|
|
"completions/min_length": 172.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 405.75,
|
|
"completions/min_terminated_length": 172.0,
|
|
"completions/max_terminated_length": 914.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2593750059604645,
|
|
"rewards/reward_func/std": 0.33622077107429504,
|
|
"reward": 0.2593750059604645,
|
|
"reward_std": 0.33622074127197266,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017580060521140695,
|
|
"sampling/sampling_logp_difference/max": 0.5126321315765381,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.1894255876541138,
|
|
"sampling/importance_sampling_ratio/max": 2.191917657852173,
|
|
"kl": 0.0007250254825521552,
|
|
"entropy": 0.034915244730655104,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.631176501512527,
|
|
"epoch": 0.00421875,
|
|
"step": 162
|
|
},
|
|
{
|
|
"loss": -0.19168990850448608,
|
|
"grad_norm": 6.406469821929932,
|
|
"learning_rate": 2.3793103448275863e-07,
|
|
"num_tokens": 1738005.0,
|
|
"completions/mean_length": 651.5,
|
|
"completions/min_length": 144.0,
|
|
"completions/max_length": 913.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 397.5,
|
|
"completions/min_terminated_length": 144.0,
|
|
"completions/max_terminated_length": 913.0,
|
|
"tools/call_frequency": 16.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.04791666939854622,
|
|
"rewards/reward_func/std": 0.1337604820728302,
|
|
"reward": 0.04791666939854622,
|
|
"reward_std": 0.1337604820728302,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015843264991417527,
|
|
"sampling/sampling_logp_difference/max": 0.4399840831756592,
|
|
"sampling/importance_sampling_ratio/min": 0.6077942252159119,
|
|
"sampling/importance_sampling_ratio/mean": 1.1782963275909424,
|
|
"sampling/importance_sampling_ratio/max": 2.306882619857788,
|
|
"kl": 0.000812459308264124,
|
|
"entropy": 0.02847969243885018,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.98835889622569,
|
|
"epoch": 0.004244791666666667,
|
|
"step": 163
|
|
},
|
|
{
|
|
"loss": -0.04339796304702759,
|
|
"grad_norm": 2.5741353034973145,
|
|
"learning_rate": 2.3620689655172413e-07,
|
|
"num_tokens": 1748108.0,
|
|
"completions/mean_length": 578.375,
|
|
"completions/min_length": 183.0,
|
|
"completions/max_length": 925.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 465.16668701171875,
|
|
"completions/min_terminated_length": 183.0,
|
|
"completions/max_terminated_length": 909.0,
|
|
"tools/call_frequency": 14.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.19374999403953552,
|
|
"rewards/reward_func/std": 0.33667224645614624,
|
|
"reward": 0.19374999403953552,
|
|
"reward_std": 0.33667227625846863,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0025493393186479807,
|
|
"sampling/sampling_logp_difference/max": 0.3548574447631836,
|
|
"sampling/importance_sampling_ratio/min": 0.3320784866809845,
|
|
"sampling/importance_sampling_ratio/mean": 1.0844089984893799,
|
|
"sampling/importance_sampling_ratio/max": 2.2945587635040283,
|
|
"kl": 0.000927839420910459,
|
|
"entropy": 0.05397579987766221,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.119193697348237,
|
|
"epoch": 0.004270833333333333,
|
|
"step": 164
|
|
},
|
|
{
|
|
"loss": -0.013513831421732903,
|
|
"grad_norm": 3.7448692321777344,
|
|
"learning_rate": 2.3448275862068964e-07,
|
|
"num_tokens": 1758033.0,
|
|
"completions/mean_length": 555.75,
|
|
"completions/min_length": 142.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 431.66668701171875,
|
|
"completions/min_terminated_length": 142.0,
|
|
"completions/max_terminated_length": 893.0,
|
|
"tools/call_frequency": 12.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2776041626930237,
|
|
"rewards/reward_func/std": 0.3195897340774536,
|
|
"reward": 0.2776041626930237,
|
|
"reward_std": 0.3195897042751312,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0024511078372597694,
|
|
"sampling/sampling_logp_difference/max": 1.265627384185791,
|
|
"sampling/importance_sampling_ratio/min": 0.7200666666030884,
|
|
"sampling/importance_sampling_ratio/mean": 1.0127418041229248,
|
|
"sampling/importance_sampling_ratio/max": 1.496435523033142,
|
|
"kl": 0.00386009416251909,
|
|
"entropy": 0.03945721045602113,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.374979108572006,
|
|
"epoch": 0.004296875,
|
|
"step": 165
|
|
},
|
|
{
|
|
"loss": 0.13043643534183502,
|
|
"grad_norm": 1.1632308959960938,
|
|
"learning_rate": 2.3275862068965515e-07,
|
|
"num_tokens": 1769603.0,
|
|
"completions/mean_length": 760.75,
|
|
"completions/min_length": 242.0,
|
|
"completions/max_length": 941.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 262.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"completions/max_terminated_length": 282.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10208333283662796,
|
|
"rewards/reward_func/std": 0.2144460529088974,
|
|
"reward": 0.10208333283662796,
|
|
"reward_std": 0.2144460380077362,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001183012267574668,
|
|
"sampling/sampling_logp_difference/max": 0.45464539527893066,
|
|
"sampling/importance_sampling_ratio/min": 0.42441806197166443,
|
|
"sampling/importance_sampling_ratio/mean": 0.8473471403121948,
|
|
"sampling/importance_sampling_ratio/max": 1.073018193244934,
|
|
"kl": 0.0005934268133387377,
|
|
"entropy": 0.021242705115582794,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.416926883161068,
|
|
"epoch": 0.004322916666666667,
|
|
"step": 166
|
|
},
|
|
{
|
|
"loss": 0.2666020095348358,
|
|
"grad_norm": 2.370063304901123,
|
|
"learning_rate": 2.3103448275862066e-07,
|
|
"num_tokens": 1780298.0,
|
|
"completions/mean_length": 651.25,
|
|
"completions/min_length": 208.0,
|
|
"completions/max_length": 922.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 403.0,
|
|
"completions/min_terminated_length": 208.0,
|
|
"completions/max_terminated_length": 920.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17569445073604584,
|
|
"rewards/reward_func/std": 0.22845789790153503,
|
|
"reward": 0.17569445073604584,
|
|
"reward_std": 0.22845789790153503,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002430615248158574,
|
|
"sampling/sampling_logp_difference/max": 0.5118851661682129,
|
|
"sampling/importance_sampling_ratio/min": 0.2852669358253479,
|
|
"sampling/importance_sampling_ratio/mean": 0.9325593709945679,
|
|
"sampling/importance_sampling_ratio/max": 1.3473148345947266,
|
|
"kl": 0.0009170873795483203,
|
|
"entropy": 0.0400894726626575,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 14.090074537321925,
|
|
"epoch": 0.004348958333333333,
|
|
"step": 167
|
|
},
|
|
{
|
|
"loss": 0.25087425112724304,
|
|
"grad_norm": 3.4152486324310303,
|
|
"learning_rate": 2.293103448275862e-07,
|
|
"num_tokens": 1790098.0,
|
|
"completions/mean_length": 540.25,
|
|
"completions/min_length": 196.0,
|
|
"completions/max_length": 999.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 485.857177734375,
|
|
"completions/min_terminated_length": 196.0,
|
|
"completions/max_terminated_length": 999.0,
|
|
"tools/call_frequency": 11.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23749999701976776,
|
|
"rewards/reward_func/std": 0.2364586740732193,
|
|
"reward": 0.23749999701976776,
|
|
"reward_std": 0.2364586740732193,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0037763570435345173,
|
|
"sampling/sampling_logp_difference/max": 0.43997764587402344,
|
|
"sampling/importance_sampling_ratio/min": 0.6774695515632629,
|
|
"sampling/importance_sampling_ratio/mean": 1.0617197751998901,
|
|
"sampling/importance_sampling_ratio/max": 1.6764647960662842,
|
|
"kl": 0.0010632717949192738,
|
|
"entropy": 0.0706241043517366,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.04471237026155,
|
|
"epoch": 0.004375,
|
|
"step": 168
|
|
},
|
|
{
|
|
"loss": -0.2610553801059723,
|
|
"grad_norm": 8.15101432800293,
|
|
"learning_rate": 2.2758620689655173e-07,
|
|
"num_tokens": 1801681.0,
|
|
"completions/mean_length": 762.375,
|
|
"completions/min_length": 172.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 493.0,
|
|
"completions/min_terminated_length": 172.0,
|
|
"completions/max_terminated_length": 924.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.07968750596046448,
|
|
"rewards/reward_func/std": 0.15381133556365967,
|
|
"reward": 0.07968750596046448,
|
|
"reward_std": 0.15381132066249847,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019949187990278006,
|
|
"sampling/sampling_logp_difference/max": 0.9378368854522705,
|
|
"sampling/importance_sampling_ratio/min": 0.516279399394989,
|
|
"sampling/importance_sampling_ratio/mean": 1.082507610321045,
|
|
"sampling/importance_sampling_ratio/max": 2.956860303878784,
|
|
"kl": 0.0006947617483774593,
|
|
"entropy": 0.03610103772371076,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.79327592253685,
|
|
"epoch": 0.004401041666666667,
|
|
"step": 169
|
|
},
|
|
{
|
|
"loss": 0.04193887114524841,
|
|
"grad_norm": 1.9980107545852661,
|
|
"learning_rate": 2.2586206896551724e-07,
|
|
"num_tokens": 1813148.0,
|
|
"completions/mean_length": 748.0,
|
|
"completions/min_length": 159.0,
|
|
"completions/max_length": 984.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 562.5,
|
|
"completions/min_terminated_length": 159.0,
|
|
"completions/max_terminated_length": 934.0,
|
|
"tools/call_frequency": 16.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23020833730697632,
|
|
"rewards/reward_func/std": 0.3250744044780731,
|
|
"reward": 0.23020833730697632,
|
|
"reward_std": 0.3250744044780731,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016748541966080666,
|
|
"sampling/sampling_logp_difference/max": 0.5591316223144531,
|
|
"sampling/importance_sampling_ratio/min": 0.3706308901309967,
|
|
"sampling/importance_sampling_ratio/mean": 1.0390164852142334,
|
|
"sampling/importance_sampling_ratio/max": 1.7724794149398804,
|
|
"kl": 0.0009891102004075947,
|
|
"entropy": 0.03096083182026632,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.78370071388781,
|
|
"epoch": 0.004427083333333333,
|
|
"step": 170
|
|
},
|
|
{
|
|
"loss": 0.4158915877342224,
|
|
"grad_norm": 2.218940496444702,
|
|
"learning_rate": 2.2413793103448274e-07,
|
|
"num_tokens": 1823123.0,
|
|
"completions/mean_length": 562.5,
|
|
"completions/min_length": 178.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 443.0,
|
|
"completions/min_terminated_length": 178.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 13.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.29131942987442017,
|
|
"rewards/reward_func/std": 0.39485064148902893,
|
|
"reward": 0.29131942987442017,
|
|
"reward_std": 0.3948506712913513,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001858189469203353,
|
|
"sampling/sampling_logp_difference/max": 0.5575137138366699,
|
|
"sampling/importance_sampling_ratio/min": 0.44172006845474243,
|
|
"sampling/importance_sampling_ratio/mean": 0.919952392578125,
|
|
"sampling/importance_sampling_ratio/max": 1.271050214767456,
|
|
"kl": 0.001865160681290945,
|
|
"entropy": 0.04240136797307059,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.87258062697947,
|
|
"epoch": 0.004453125,
|
|
"step": 171
|
|
},
|
|
{
|
|
"loss": 0.06321065127849579,
|
|
"grad_norm": 3.823612928390503,
|
|
"learning_rate": 2.2241379310344828e-07,
|
|
"num_tokens": 1831958.0,
|
|
"completions/mean_length": 419.5,
|
|
"completions/min_length": 187.0,
|
|
"completions/max_length": 899.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 351.0000305175781,
|
|
"completions/min_terminated_length": 187.0,
|
|
"completions/max_terminated_length": 888.0,
|
|
"tools/call_frequency": 11.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2593750059604645,
|
|
"rewards/reward_func/std": 0.3990921974182129,
|
|
"reward": 0.2593750059604645,
|
|
"reward_std": 0.3990921974182129,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0030535650439560413,
|
|
"sampling/sampling_logp_difference/max": 0.7464065551757812,
|
|
"sampling/importance_sampling_ratio/min": 0.5985996127128601,
|
|
"sampling/importance_sampling_ratio/mean": 1.1454390287399292,
|
|
"sampling/importance_sampling_ratio/max": 1.9490469694137573,
|
|
"kl": 0.0017178901434817817,
|
|
"entropy": 0.05655999749433249,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.417142949998379,
|
|
"epoch": 0.004479166666666667,
|
|
"step": 172
|
|
},
|
|
{
|
|
"loss": 0.049227386713027954,
|
|
"grad_norm": 2.150346517562866,
|
|
"learning_rate": 2.206896551724138e-07,
|
|
"num_tokens": 1843363.0,
|
|
"completions/mean_length": 741.25,
|
|
"completions/min_length": 175.0,
|
|
"completions/max_length": 939.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 718.7142944335938,
|
|
"completions/min_terminated_length": 175.0,
|
|
"completions/max_terminated_length": 939.0,
|
|
"tools/call_frequency": 18.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.22812500596046448,
|
|
"rewards/reward_func/std": 0.38018736243247986,
|
|
"reward": 0.22812500596046448,
|
|
"reward_std": 0.38018733263015747,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015395215013995767,
|
|
"sampling/sampling_logp_difference/max": 0.9508585929870605,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.992497980594635,
|
|
"sampling/importance_sampling_ratio/max": 1.61998450756073,
|
|
"kl": 0.0013814405247103423,
|
|
"entropy": 0.02334262226941064,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.807927025482059,
|
|
"epoch": 0.004505208333333333,
|
|
"step": 173
|
|
},
|
|
{
|
|
"loss": 0.2290850579738617,
|
|
"grad_norm": 2.2492990493774414,
|
|
"learning_rate": 2.189655172413793e-07,
|
|
"num_tokens": 1854260.0,
|
|
"completions/mean_length": 677.0,
|
|
"completions/min_length": 178.0,
|
|
"completions/max_length": 993.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 519.0,
|
|
"completions/min_terminated_length": 178.0,
|
|
"completions/max_terminated_length": 993.0,
|
|
"tools/call_frequency": 15.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.19583334028720856,
|
|
"rewards/reward_func/std": 0.32425129413604736,
|
|
"reward": 0.19583334028720856,
|
|
"reward_std": 0.324251264333725,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020815751049667597,
|
|
"sampling/sampling_logp_difference/max": 0.44968223571777344,
|
|
"sampling/importance_sampling_ratio/min": 0.6338522434234619,
|
|
"sampling/importance_sampling_ratio/mean": 0.8647899627685547,
|
|
"sampling/importance_sampling_ratio/max": 1.2204031944274902,
|
|
"kl": 0.0011502038778417045,
|
|
"entropy": 0.0401098444708623,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.562533278018236,
|
|
"epoch": 0.00453125,
|
|
"step": 174
|
|
},
|
|
{
|
|
"loss": 0.08516474068164825,
|
|
"grad_norm": 1.770313024520874,
|
|
"learning_rate": 2.172413793103448e-07,
|
|
"num_tokens": 1865742.0,
|
|
"completions/mean_length": 750.5,
|
|
"completions/min_length": 178.0,
|
|
"completions/max_length": 978.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 227.5,
|
|
"completions/min_terminated_length": 178.0,
|
|
"completions/max_terminated_length": 277.0,
|
|
"tools/call_frequency": 17.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.12968750298023224,
|
|
"rewards/reward_func/std": 0.20088393986225128,
|
|
"reward": 0.12968750298023224,
|
|
"reward_std": 0.2008839249610901,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0011201961897313595,
|
|
"sampling/sampling_logp_difference/max": 0.5009331703186035,
|
|
"sampling/importance_sampling_ratio/min": 0.5202377438545227,
|
|
"sampling/importance_sampling_ratio/mean": 1.076450228691101,
|
|
"sampling/importance_sampling_ratio/max": 1.9205222129821777,
|
|
"kl": 0.0004573204644202633,
|
|
"entropy": 0.029519025003537536,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.89030877314508,
|
|
"epoch": 0.004557291666666667,
|
|
"step": 175
|
|
},
|
|
{
|
|
"loss": 0.22880898416042328,
|
|
"grad_norm": 3.314643383026123,
|
|
"learning_rate": 2.155172413793103e-07,
|
|
"num_tokens": 1875905.0,
|
|
"completions/mean_length": 584.375,
|
|
"completions/min_length": 194.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 247.25,
|
|
"completions/min_terminated_length": 194.0,
|
|
"completions/max_terminated_length": 301.0,
|
|
"tools/call_frequency": 13.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1848958432674408,
|
|
"rewards/reward_func/std": 0.3011217415332794,
|
|
"reward": 0.1848958432674408,
|
|
"reward_std": 0.3011217415332794,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0028747795149683952,
|
|
"sampling/sampling_logp_difference/max": 0.4437451958656311,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.034279227256775,
|
|
"sampling/importance_sampling_ratio/max": 2.0627996921539307,
|
|
"kl": 0.0009320553981524426,
|
|
"entropy": 0.05926871777046472,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.51762349717319,
|
|
"epoch": 0.004583333333333333,
|
|
"step": 176
|
|
},
|
|
{
|
|
"loss": 0.07345214486122131,
|
|
"grad_norm": 1.2291994094848633,
|
|
"learning_rate": 2.1379310344827587e-07,
|
|
"num_tokens": 1887238.0,
|
|
"completions/mean_length": 732.5,
|
|
"completions/min_length": 258.0,
|
|
"completions/max_length": 944.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 669.6666870117188,
|
|
"completions/min_terminated_length": 258.0,
|
|
"completions/max_terminated_length": 914.0,
|
|
"tools/call_frequency": 18.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23333334922790527,
|
|
"rewards/reward_func/std": 0.2555728256702423,
|
|
"reward": 0.23333334922790527,
|
|
"reward_std": 0.2555727958679199,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016446438385173678,
|
|
"sampling/sampling_logp_difference/max": 0.5969643592834473,
|
|
"sampling/importance_sampling_ratio/min": 0.4623764157295227,
|
|
"sampling/importance_sampling_ratio/mean": 0.6544597148895264,
|
|
"sampling/importance_sampling_ratio/max": 0.8233513832092285,
|
|
"kl": 0.0005599233002158144,
|
|
"entropy": 0.02918639045674354,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.874919867143035,
|
|
"epoch": 0.004609375,
|
|
"step": 177
|
|
},
|
|
{
|
|
"loss": -0.25620150566101074,
|
|
"grad_norm": 1.5966987609863281,
|
|
"learning_rate": 2.1206896551724138e-07,
|
|
"num_tokens": 1897953.0,
|
|
"completions/mean_length": 654.25,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 495.6000061035156,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 922.0,
|
|
"tools/call_frequency": 15.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13437500596046448,
|
|
"rewards/reward_func/std": 0.2761138379573822,
|
|
"reward": 0.13437500596046448,
|
|
"reward_std": 0.2761138379573822,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015854177763685584,
|
|
"sampling/sampling_logp_difference/max": 0.6636886596679688,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.6361932158470154,
|
|
"sampling/importance_sampling_ratio/max": 1.1001241207122803,
|
|
"kl": 0.00038644070855298196,
|
|
"entropy": 0.026699737587478012,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.498098915442824,
|
|
"epoch": 0.004635416666666667,
|
|
"step": 178
|
|
},
|
|
{
|
|
"loss": 0.15780450403690338,
|
|
"grad_norm": 2.487543821334839,
|
|
"learning_rate": 2.103448275862069e-07,
|
|
"num_tokens": 1908680.0,
|
|
"completions/mean_length": 655.25,
|
|
"completions/min_length": 175.0,
|
|
"completions/max_length": 945.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 511.6000061035156,
|
|
"completions/min_terminated_length": 175.0,
|
|
"completions/max_terminated_length": 945.0,
|
|
"tools/call_frequency": 16.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1875,
|
|
"rewards/reward_func/std": 0.32348546385765076,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.32348549365997314,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001574549707584083,
|
|
"sampling/sampling_logp_difference/max": 0.5479118824005127,
|
|
"sampling/importance_sampling_ratio/min": 0.4904833734035492,
|
|
"sampling/importance_sampling_ratio/mean": 0.8853148221969604,
|
|
"sampling/importance_sampling_ratio/max": 1.403978943824768,
|
|
"kl": 0.0007800783043876436,
|
|
"entropy": 0.037897152767982334,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.465267417952418,
|
|
"epoch": 0.004661458333333333,
|
|
"step": 179
|
|
},
|
|
{
|
|
"loss": -0.225298210978508,
|
|
"grad_norm": 1.859980583190918,
|
|
"learning_rate": 2.086206896551724e-07,
|
|
"num_tokens": 1920069.0,
|
|
"completions/mean_length": 737.125,
|
|
"completions/min_length": 139.0,
|
|
"completions/max_length": 990.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 677.6666870117188,
|
|
"completions/min_terminated_length": 139.0,
|
|
"completions/max_terminated_length": 990.0,
|
|
"tools/call_frequency": 16.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.078125,
|
|
"rewards/reward_func/std": 0.2118951380252838,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.211895152926445,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016199363162741065,
|
|
"sampling/sampling_logp_difference/max": 1.0769484043121338,
|
|
"sampling/importance_sampling_ratio/min": 0.1269417107105255,
|
|
"sampling/importance_sampling_ratio/mean": 0.8629543781280518,
|
|
"sampling/importance_sampling_ratio/max": 2.2890443801879883,
|
|
"kl": 0.0007484557736461284,
|
|
"entropy": 0.02678147971164435,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.927622143179178,
|
|
"epoch": 0.0046875,
|
|
"step": 180
|
|
},
|
|
{
|
|
"loss": 0.11200682073831558,
|
|
"grad_norm": 3.027215003967285,
|
|
"learning_rate": 2.0689655172413793e-07,
|
|
"num_tokens": 1929627.0,
|
|
"completions/mean_length": 509.875,
|
|
"completions/min_length": 223.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 263.3999938964844,
|
|
"completions/min_terminated_length": 223.0,
|
|
"completions/max_terminated_length": 340.0,
|
|
"tools/call_frequency": 12.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10520833730697632,
|
|
"rewards/reward_func/std": 0.2627290189266205,
|
|
"reward": 0.10520833730697632,
|
|
"reward_std": 0.2627290189266205,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002961767604574561,
|
|
"sampling/sampling_logp_difference/max": 0.3726496696472168,
|
|
"sampling/importance_sampling_ratio/min": 0.5488517880439758,
|
|
"sampling/importance_sampling_ratio/mean": 1.050106406211853,
|
|
"sampling/importance_sampling_ratio/max": 1.8311142921447754,
|
|
"kl": 0.0010003545794461388,
|
|
"entropy": 0.06158387000323273,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.692367250099778,
|
|
"epoch": 0.004713541666666667,
|
|
"step": 181
|
|
},
|
|
{
|
|
"loss": 0.12553197145462036,
|
|
"grad_norm": 1.624507188796997,
|
|
"learning_rate": 2.0517241379310344e-07,
|
|
"num_tokens": 1938948.0,
|
|
"completions/mean_length": 481.125,
|
|
"completions/min_length": 170.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 215.60000610351562,
|
|
"completions/min_terminated_length": 170.0,
|
|
"completions/max_terminated_length": 276.0,
|
|
"tools/call_frequency": 11.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1145833358168602,
|
|
"rewards/reward_func/std": 0.2881850302219391,
|
|
"reward": 0.1145833358168602,
|
|
"reward_std": 0.2881850004196167,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0026484474074095488,
|
|
"sampling/sampling_logp_difference/max": 0.7916420698165894,
|
|
"sampling/importance_sampling_ratio/min": 0.2649141550064087,
|
|
"sampling/importance_sampling_ratio/mean": 1.0132482051849365,
|
|
"sampling/importance_sampling_ratio/max": 1.6027836799621582,
|
|
"kl": 0.0009807003916648682,
|
|
"entropy": 0.049312864197418094,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.518522595986724,
|
|
"epoch": 0.0047395833333333335,
|
|
"step": 182
|
|
},
|
|
{
|
|
"loss": 0.233491450548172,
|
|
"grad_norm": 1.9921684265136719,
|
|
"learning_rate": 2.0344827586206895e-07,
|
|
"num_tokens": 1949004.0,
|
|
"completions/mean_length": 571.0,
|
|
"completions/min_length": 171.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 367.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"completions/max_terminated_length": 932.0,
|
|
"tools/call_frequency": 13.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13159722089767456,
|
|
"rewards/reward_func/std": 0.25972673296928406,
|
|
"reward": 0.13159722089767456,
|
|
"reward_std": 0.25972673296928406,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002537477994337678,
|
|
"sampling/sampling_logp_difference/max": 0.6249924898147583,
|
|
"sampling/importance_sampling_ratio/min": 0.11459740996360779,
|
|
"sampling/importance_sampling_ratio/mean": 0.9119467735290527,
|
|
"sampling/importance_sampling_ratio/max": 1.458534836769104,
|
|
"kl": 0.0015212330182521328,
|
|
"entropy": 0.05697890225565061,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.995790094137192,
|
|
"epoch": 0.004765625,
|
|
"step": 183
|
|
},
|
|
{
|
|
"loss": -0.04471494257450104,
|
|
"grad_norm": 3.49534010887146,
|
|
"learning_rate": 2.0172413793103446e-07,
|
|
"num_tokens": 1959790.0,
|
|
"completions/mean_length": 662.875,
|
|
"completions/min_length": 185.0,
|
|
"completions/max_length": 973.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 503.3999938964844,
|
|
"completions/min_terminated_length": 185.0,
|
|
"completions/max_terminated_length": 973.0,
|
|
"tools/call_frequency": 16.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.02968749962747097,
|
|
"rewards/reward_func/std": 0.09304796904325485,
|
|
"reward": 0.02968749962747097,
|
|
"reward_std": 0.09304797649383545,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016616502543911338,
|
|
"sampling/sampling_logp_difference/max": 0.9263906478881836,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.9134335517883301,
|
|
"sampling/importance_sampling_ratio/max": 2.5355582237243652,
|
|
"kl": 0.000720762160653976,
|
|
"entropy": 0.035813313676044345,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.29391523078084,
|
|
"epoch": 0.004791666666666666,
|
|
"step": 184
|
|
},
|
|
{
|
|
"loss": -0.07143831998109818,
|
|
"grad_norm": 1.9316093921661377,
|
|
"learning_rate": 2e-07,
|
|
"num_tokens": 1968945.0,
|
|
"completions/mean_length": 458.125,
|
|
"completions/min_length": 124.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 179.8000030517578,
|
|
"completions/min_terminated_length": 124.0,
|
|
"completions/max_terminated_length": 254.0,
|
|
"tools/call_frequency": 10.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.24079862236976624,
|
|
"rewards/reward_func/std": 0.40491288900375366,
|
|
"reward": 0.24079862236976624,
|
|
"reward_std": 0.40491288900375366,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022107826080173254,
|
|
"sampling/sampling_logp_difference/max": 0.4576582908630371,
|
|
"sampling/importance_sampling_ratio/min": 0.5392160415649414,
|
|
"sampling/importance_sampling_ratio/mean": 0.7929911613464355,
|
|
"sampling/importance_sampling_ratio/max": 1.0272767543792725,
|
|
"kl": 0.0010238880349788815,
|
|
"entropy": 0.04513480397872627,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.134196817874908,
|
|
"epoch": 0.0048177083333333336,
|
|
"step": 185
|
|
},
|
|
{
|
|
"loss": -0.12958799302577972,
|
|
"grad_norm": 3.0536065101623535,
|
|
"learning_rate": 1.9827586206896553e-07,
|
|
"num_tokens": 1979645.0,
|
|
"completions/mean_length": 652.5,
|
|
"completions/min_length": 153.0,
|
|
"completions/max_length": 942.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 510.6000061035156,
|
|
"completions/min_terminated_length": 153.0,
|
|
"completions/max_terminated_length": 932.0,
|
|
"tools/call_frequency": 15.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18020834028720856,
|
|
"rewards/reward_func/std": 0.2568868100643158,
|
|
"reward": 0.18020834028720856,
|
|
"reward_std": 0.2568867802619934,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002992450026795268,
|
|
"sampling/sampling_logp_difference/max": 1.0369462966918945,
|
|
"sampling/importance_sampling_ratio/min": 0.11431094259023666,
|
|
"sampling/importance_sampling_ratio/mean": 1.1303011178970337,
|
|
"sampling/importance_sampling_ratio/max": 2.2890982627868652,
|
|
"kl": 0.0011318122033117106,
|
|
"entropy": 0.04861332650762051,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.32558373361826,
|
|
"epoch": 0.00484375,
|
|
"step": 186
|
|
},
|
|
{
|
|
"loss": -0.07571446895599365,
|
|
"grad_norm": 1.7993505001068115,
|
|
"learning_rate": 1.9655172413793103e-07,
|
|
"num_tokens": 1990416.0,
|
|
"completions/mean_length": 661.0,
|
|
"completions/min_length": 236.0,
|
|
"completions/max_length": 941.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 527.7999877929688,
|
|
"completions/min_terminated_length": 236.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 16.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10520832985639572,
|
|
"rewards/reward_func/std": 0.23700083792209625,
|
|
"reward": 0.10520832985639572,
|
|
"reward_std": 0.23700083792209625,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015399199910461903,
|
|
"sampling/sampling_logp_difference/max": 0.5872076749801636,
|
|
"sampling/importance_sampling_ratio/min": 0.6412323117256165,
|
|
"sampling/importance_sampling_ratio/mean": 0.9178242683410645,
|
|
"sampling/importance_sampling_ratio/max": 1.473846673965454,
|
|
"kl": 0.0006083876141929068,
|
|
"entropy": 0.03189178236061707,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.348942197859287,
|
|
"epoch": 0.004869791666666666,
|
|
"step": 187
|
|
},
|
|
{
|
|
"loss": 0.026084240525960922,
|
|
"grad_norm": 3.0957858562469482,
|
|
"learning_rate": 1.9482758620689654e-07,
|
|
"num_tokens": 1998305.0,
|
|
"completions/mean_length": 301.375,
|
|
"completions/min_length": 145.0,
|
|
"completions/max_length": 964.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 206.71429443359375,
|
|
"completions/min_terminated_length": 145.0,
|
|
"completions/max_terminated_length": 297.0,
|
|
"tools/call_frequency": 7.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": -0.00937500037252903,
|
|
"rewards/reward_func/std": 0.202633336186409,
|
|
"reward": -0.00937500037252903,
|
|
"reward_std": 0.202633336186409,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.005483698565512896,
|
|
"sampling/sampling_logp_difference/max": 1.4890155792236328,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.6771097779273987,
|
|
"sampling/importance_sampling_ratio/max": 1.0791782140731812,
|
|
"kl": 0.0010755132889244123,
|
|
"entropy": 0.06160821416415274,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.24883876554668,
|
|
"epoch": 0.004895833333333334,
|
|
"step": 188
|
|
},
|
|
{
|
|
"loss": -0.03178441524505615,
|
|
"grad_norm": 5.3132171630859375,
|
|
"learning_rate": 1.9310344827586205e-07,
|
|
"num_tokens": 2007723.0,
|
|
"completions/mean_length": 493.375,
|
|
"completions/min_length": 226.0,
|
|
"completions/max_length": 919.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 358.16668701171875,
|
|
"completions/min_terminated_length": 226.0,
|
|
"completions/max_terminated_length": 904.0,
|
|
"tools/call_frequency": 11.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.30863094329833984,
|
|
"rewards/reward_func/std": 0.3565838932991028,
|
|
"reward": 0.30863094329833984,
|
|
"reward_std": 0.3565838932991028,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002891829004511237,
|
|
"sampling/sampling_logp_difference/max": 0.8391867876052856,
|
|
"sampling/importance_sampling_ratio/min": 0.3809032440185547,
|
|
"sampling/importance_sampling_ratio/mean": 1.1593118906021118,
|
|
"sampling/importance_sampling_ratio/max": 2.3184940814971924,
|
|
"kl": 0.003589209065467003,
|
|
"entropy": 0.041746810311451554,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.068575026467443,
|
|
"epoch": 0.004921875,
|
|
"step": 189
|
|
},
|
|
{
|
|
"loss": -0.05935106426477432,
|
|
"grad_norm": 1.9522041082382202,
|
|
"learning_rate": 1.9137931034482756e-07,
|
|
"num_tokens": 2018509.0,
|
|
"completions/mean_length": 662.75,
|
|
"completions/min_length": 234.0,
|
|
"completions/max_length": 941.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 517.2000122070312,
|
|
"completions/min_terminated_length": 234.0,
|
|
"completions/max_terminated_length": 930.0,
|
|
"tools/call_frequency": 15.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3489583432674408,
|
|
"rewards/reward_func/std": 0.3647797107696533,
|
|
"reward": 0.3489583432674408,
|
|
"reward_std": 0.3647797107696533,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019989614374935627,
|
|
"sampling/sampling_logp_difference/max": 0.516481876373291,
|
|
"sampling/importance_sampling_ratio/min": 0.48473331332206726,
|
|
"sampling/importance_sampling_ratio/mean": 0.858253002166748,
|
|
"sampling/importance_sampling_ratio/max": 1.4411009550094604,
|
|
"kl": 0.001979286225832766,
|
|
"entropy": 0.04130327369784936,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.638440130278468,
|
|
"epoch": 0.0049479166666666664,
|
|
"step": 190
|
|
},
|
|
{
|
|
"loss": -0.16264355182647705,
|
|
"grad_norm": 2.1364755630493164,
|
|
"learning_rate": 1.896551724137931e-07,
|
|
"num_tokens": 2029886.0,
|
|
"completions/mean_length": 737.875,
|
|
"completions/min_length": 209.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 614.2000122070312,
|
|
"completions/min_terminated_length": 209.0,
|
|
"completions/max_terminated_length": 889.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23333331942558289,
|
|
"rewards/reward_func/std": 0.3934231400489807,
|
|
"reward": 0.23333331942558289,
|
|
"reward_std": 0.3934231400489807,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016281601274386048,
|
|
"sampling/sampling_logp_difference/max": 0.6081318855285645,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8818901181221008,
|
|
"sampling/importance_sampling_ratio/max": 1.7482553720474243,
|
|
"kl": 0.0012435931739673833,
|
|
"entropy": 0.01885169354500249,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.973266150802374,
|
|
"epoch": 0.004973958333333334,
|
|
"step": 191
|
|
},
|
|
{
|
|
"loss": -0.006880059838294983,
|
|
"grad_norm": 2.0013539791107178,
|
|
"learning_rate": 1.879310344827586e-07,
|
|
"num_tokens": 2040483.0,
|
|
"completions/mean_length": 638.875,
|
|
"completions/min_length": 158.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 471.0,
|
|
"completions/min_terminated_length": 158.0,
|
|
"completions/max_terminated_length": 929.0,
|
|
"tools/call_frequency": 14.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.171875,
|
|
"rewards/reward_func/std": 0.3304319679737091,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.3304319381713867,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0012600580230355263,
|
|
"sampling/sampling_logp_difference/max": 0.34318965673446655,
|
|
"sampling/importance_sampling_ratio/min": 0.6803224086761475,
|
|
"sampling/importance_sampling_ratio/mean": 0.9460889101028442,
|
|
"sampling/importance_sampling_ratio/max": 1.2033910751342773,
|
|
"kl": 0.0015528129920312495,
|
|
"entropy": 0.026286052190698683,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.46360027603805,
|
|
"epoch": 0.005,
|
|
"step": 192
|
|
},
|
|
{
|
|
"loss": -0.1957259625196457,
|
|
"grad_norm": 1.8083112239837646,
|
|
"learning_rate": 1.8620689655172414e-07,
|
|
"num_tokens": 2051777.0,
|
|
"completions/mean_length": 727.625,
|
|
"completions/min_length": 203.0,
|
|
"completions/max_length": 923.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 551.5,
|
|
"completions/min_terminated_length": 203.0,
|
|
"completions/max_terminated_length": 892.0,
|
|
"tools/call_frequency": 17.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.078125,
|
|
"rewards/reward_func/std": 0.2118951380252838,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2118951380252838,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014417878119274974,
|
|
"sampling/sampling_logp_difference/max": 0.42551088333129883,
|
|
"sampling/importance_sampling_ratio/min": 0.41692665219306946,
|
|
"sampling/importance_sampling_ratio/mean": 0.7611578702926636,
|
|
"sampling/importance_sampling_ratio/max": 1.2033318281173706,
|
|
"kl": 0.001736071409595752,
|
|
"entropy": 0.022657594992779195,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.40371017344296,
|
|
"epoch": 0.0050260416666666665,
|
|
"step": 193
|
|
},
|
|
{
|
|
"loss": 0.23657983541488647,
|
|
"grad_norm": 1.7275878190994263,
|
|
"learning_rate": 1.8448275862068964e-07,
|
|
"num_tokens": 2061941.0,
|
|
"completions/mean_length": 585.25,
|
|
"completions/min_length": 155.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 378.6000061035156,
|
|
"completions/min_terminated_length": 155.0,
|
|
"completions/max_terminated_length": 900.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.20208331942558289,
|
|
"rewards/reward_func/std": 0.3099779188632965,
|
|
"reward": 0.20208331942558289,
|
|
"reward_std": 0.3099779188632965,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002642299048602581,
|
|
"sampling/sampling_logp_difference/max": 0.5885645151138306,
|
|
"sampling/importance_sampling_ratio/min": 0.6022801399230957,
|
|
"sampling/importance_sampling_ratio/mean": 0.857795238494873,
|
|
"sampling/importance_sampling_ratio/max": 1.2797785997390747,
|
|
"kl": 0.0008649896226415876,
|
|
"entropy": 0.04714253213023767,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.035484120249748,
|
|
"epoch": 0.005052083333333333,
|
|
"step": 194
|
|
},
|
|
{
|
|
"loss": 0.30008426308631897,
|
|
"grad_norm": 1.090652346611023,
|
|
"learning_rate": 1.8275862068965518e-07,
|
|
"num_tokens": 2073348.0,
|
|
"completions/mean_length": 741.5,
|
|
"completions/min_length": 209.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 568.75,
|
|
"completions/min_terminated_length": 209.0,
|
|
"completions/max_terminated_length": 943.0,
|
|
"tools/call_frequency": 18.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18854168057441711,
|
|
"rewards/reward_func/std": 0.2694401144981384,
|
|
"reward": 0.18854168057441711,
|
|
"reward_std": 0.26944008469581604,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014291537227109075,
|
|
"sampling/sampling_logp_difference/max": 0.40829479694366455,
|
|
"sampling/importance_sampling_ratio/min": 0.498983234167099,
|
|
"sampling/importance_sampling_ratio/mean": 0.8201120495796204,
|
|
"sampling/importance_sampling_ratio/max": 1.2254959344863892,
|
|
"kl": 0.0012856879839091562,
|
|
"entropy": 0.03303621069062501,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.834920659661293,
|
|
"epoch": 0.005078125,
|
|
"step": 195
|
|
},
|
|
{
|
|
"loss": 0.2593111991882324,
|
|
"grad_norm": 1.4542943239212036,
|
|
"learning_rate": 1.810344827586207e-07,
|
|
"num_tokens": 2084870.0,
|
|
"completions/mean_length": 754.375,
|
|
"completions/min_length": 271.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 281.0,
|
|
"completions/min_terminated_length": 271.0,
|
|
"completions/max_terminated_length": 291.0,
|
|
"tools/call_frequency": 18.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17239584028720856,
|
|
"rewards/reward_func/std": 0.23310936987400055,
|
|
"reward": 0.17239584028720856,
|
|
"reward_std": 0.23310934007167816,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018010105704888701,
|
|
"sampling/sampling_logp_difference/max": 0.4082871675491333,
|
|
"sampling/importance_sampling_ratio/min": 0.3525625765323639,
|
|
"sampling/importance_sampling_ratio/mean": 0.6324527859687805,
|
|
"sampling/importance_sampling_ratio/max": 1.3656251430511475,
|
|
"kl": 0.0008730947242838738,
|
|
"entropy": 0.03029084310401231,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.814585618674755,
|
|
"epoch": 0.005104166666666667,
|
|
"step": 196
|
|
},
|
|
{
|
|
"loss": 0.12667465209960938,
|
|
"grad_norm": 4.130545139312744,
|
|
"learning_rate": 1.793103448275862e-07,
|
|
"num_tokens": 2094569.0,
|
|
"completions/mean_length": 525.75,
|
|
"completions/min_length": 190.0,
|
|
"completions/max_length": 957.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 279.6000061035156,
|
|
"completions/min_terminated_length": 190.0,
|
|
"completions/max_terminated_length": 331.0,
|
|
"tools/call_frequency": 12.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17760416865348816,
|
|
"rewards/reward_func/std": 0.22652390599250793,
|
|
"reward": 0.17760416865348816,
|
|
"reward_std": 0.22652387619018555,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0034457731526345015,
|
|
"sampling/sampling_logp_difference/max": 1.2233293056488037,
|
|
"sampling/importance_sampling_ratio/min": 0.17829366028308868,
|
|
"sampling/importance_sampling_ratio/mean": 0.8644810318946838,
|
|
"sampling/importance_sampling_ratio/max": 1.7935373783111572,
|
|
"kl": 0.0021706958386857877,
|
|
"entropy": 0.06500509916804731,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.447942789644003,
|
|
"epoch": 0.005130208333333333,
|
|
"step": 197
|
|
},
|
|
{
|
|
"loss": 0.13994529843330383,
|
|
"grad_norm": 2.5361225605010986,
|
|
"learning_rate": 1.775862068965517e-07,
|
|
"num_tokens": 2105269.0,
|
|
"completions/mean_length": 652.625,
|
|
"completions/min_length": 234.0,
|
|
"completions/max_length": 912.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 508.3999938964844,
|
|
"completions/min_terminated_length": 234.0,
|
|
"completions/max_terminated_length": 908.0,
|
|
"tools/call_frequency": 16.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23874998092651367,
|
|
"rewards/reward_func/std": 0.31492093205451965,
|
|
"reward": 0.23874998092651367,
|
|
"reward_std": 0.31492090225219727,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017684712074697018,
|
|
"sampling/sampling_logp_difference/max": 0.7780963182449341,
|
|
"sampling/importance_sampling_ratio/min": 0.42366376519203186,
|
|
"sampling/importance_sampling_ratio/mean": 0.8419467806816101,
|
|
"sampling/importance_sampling_ratio/max": 1.1138273477554321,
|
|
"kl": 0.0012041696113840317,
|
|
"entropy": 0.04688376575359143,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.241034708917141,
|
|
"epoch": 0.00515625,
|
|
"step": 198
|
|
},
|
|
{
|
|
"loss": 0.22572314739227295,
|
|
"grad_norm": 1.5653637647628784,
|
|
"learning_rate": 1.758620689655172e-07,
|
|
"num_tokens": 2116009.0,
|
|
"completions/mean_length": 656.25,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 934.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 222.33334350585938,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 269.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13749998807907104,
|
|
"rewards/reward_func/std": 0.2150581330060959,
|
|
"reward": 0.13749998807907104,
|
|
"reward_std": 0.2150581181049347,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015116354916244745,
|
|
"sampling/sampling_logp_difference/max": 0.6488045454025269,
|
|
"sampling/importance_sampling_ratio/min": 0.4345998466014862,
|
|
"sampling/importance_sampling_ratio/mean": 0.8196628093719482,
|
|
"sampling/importance_sampling_ratio/max": 1.1362711191177368,
|
|
"kl": 0.0006651139240148041,
|
|
"entropy": 0.02626343915471807,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.187324862927198,
|
|
"epoch": 0.005182291666666667,
|
|
"step": 199
|
|
},
|
|
{
|
|
"loss": -0.02718445658683777,
|
|
"grad_norm": 1.9352409839630127,
|
|
"learning_rate": 1.7413793103448275e-07,
|
|
"num_tokens": 2128157.0,
|
|
"completions/mean_length": 832.125,
|
|
"completions/min_length": 301.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 688.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"completions/max_terminated_length": 897.0,
|
|
"tools/call_frequency": 18.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.30416667461395264,
|
|
"rewards/reward_func/std": 0.27738863229751587,
|
|
"reward": 0.30416667461395264,
|
|
"reward_std": 0.27738863229751587,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001453466946259141,
|
|
"sampling/sampling_logp_difference/max": 0.5373185873031616,
|
|
"sampling/importance_sampling_ratio/min": 0.37720996141433716,
|
|
"sampling/importance_sampling_ratio/mean": 0.7281069755554199,
|
|
"sampling/importance_sampling_ratio/max": 1.0499807596206665,
|
|
"kl": 0.0007409485494918044,
|
|
"entropy": 0.02756094903452322,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.891134859994054,
|
|
"epoch": 0.005208333333333333,
|
|
"step": 200
|
|
},
|
|
{
|
|
"loss": 0.10805980116128922,
|
|
"grad_norm": 2.162609338760376,
|
|
"learning_rate": 1.7241379310344828e-07,
|
|
"num_tokens": 2139652.0,
|
|
"completions/mean_length": 751.0,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 460.3333435058594,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 915.0,
|
|
"tools/call_frequency": 17.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1106249988079071,
|
|
"rewards/reward_func/std": 0.23794414103031158,
|
|
"reward": 0.1106249988079071,
|
|
"reward_std": 0.23794414103031158,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015850977506488562,
|
|
"sampling/sampling_logp_difference/max": 0.5478174686431885,
|
|
"sampling/importance_sampling_ratio/min": 0.6412765383720398,
|
|
"sampling/importance_sampling_ratio/mean": 0.834298312664032,
|
|
"sampling/importance_sampling_ratio/max": 1.1711316108703613,
|
|
"kl": 0.001043963631218503,
|
|
"entropy": 0.03238698418135755,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.423177126795053,
|
|
"epoch": 0.005234375,
|
|
"step": 201
|
|
},
|
|
{
|
|
"loss": 0.2655937075614929,
|
|
"grad_norm": 1.5397539138793945,
|
|
"learning_rate": 1.706896551724138e-07,
|
|
"num_tokens": 2151787.0,
|
|
"completions/mean_length": 831.875,
|
|
"completions/min_length": 243.0,
|
|
"completions/max_length": 933.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 690.3333740234375,
|
|
"completions/min_terminated_length": 243.0,
|
|
"completions/max_terminated_length": 933.0,
|
|
"tools/call_frequency": 19.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1354166716337204,
|
|
"rewards/reward_func/std": 0.2415948212146759,
|
|
"reward": 0.1354166716337204,
|
|
"reward_std": 0.2415948212146759,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016493527218699455,
|
|
"sampling/sampling_logp_difference/max": 1.1376235485076904,
|
|
"sampling/importance_sampling_ratio/min": 0.3922761082649231,
|
|
"sampling/importance_sampling_ratio/mean": 0.8968245983123779,
|
|
"sampling/importance_sampling_ratio/max": 1.6307247877120972,
|
|
"kl": 0.0005017550149659655,
|
|
"entropy": 0.02153569128131494,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.782801426947117,
|
|
"epoch": 0.005260416666666667,
|
|
"step": 202
|
|
},
|
|
{
|
|
"loss": 0.14084240794181824,
|
|
"grad_norm": 2.759458541870117,
|
|
"learning_rate": 1.689655172413793e-07,
|
|
"num_tokens": 2161887.0,
|
|
"completions/mean_length": 577.625,
|
|
"completions/min_length": 189.0,
|
|
"completions/max_length": 933.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 463.8333435058594,
|
|
"completions/min_terminated_length": 189.0,
|
|
"completions/max_terminated_length": 933.0,
|
|
"tools/call_frequency": 13.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1666666716337204,
|
|
"rewards/reward_func/std": 0.27788886427879333,
|
|
"reward": 0.1666666716337204,
|
|
"reward_std": 0.27788886427879333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002200394868850708,
|
|
"sampling/sampling_logp_difference/max": 0.4887423515319824,
|
|
"sampling/importance_sampling_ratio/min": 0.32323095202445984,
|
|
"sampling/importance_sampling_ratio/mean": 0.991737425327301,
|
|
"sampling/importance_sampling_ratio/max": 1.999720811843872,
|
|
"kl": 0.0009838556693466671,
|
|
"entropy": 0.04835216552601196,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.602987168356776,
|
|
"epoch": 0.005286458333333333,
|
|
"step": 203
|
|
},
|
|
{
|
|
"loss": -0.024787144735455513,
|
|
"grad_norm": 1.7155511379241943,
|
|
"learning_rate": 1.6724137931034483e-07,
|
|
"num_tokens": 2171998.0,
|
|
"completions/mean_length": 578.0,
|
|
"completions/min_length": 169.0,
|
|
"completions/max_length": 966.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 359.20001220703125,
|
|
"completions/min_terminated_length": 169.0,
|
|
"completions/max_terminated_length": 831.0,
|
|
"tools/call_frequency": 14.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.04218750074505806,
|
|
"rewards/reward_func/std": 0.1761362999677658,
|
|
"reward": 0.04218750074505806,
|
|
"reward_std": 0.176136314868927,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0021027475595474243,
|
|
"sampling/sampling_logp_difference/max": 0.5611553192138672,
|
|
"sampling/importance_sampling_ratio/min": 0.27344444394111633,
|
|
"sampling/importance_sampling_ratio/mean": 0.8993446230888367,
|
|
"sampling/importance_sampling_ratio/max": 1.5894403457641602,
|
|
"kl": 0.0020877326616073333,
|
|
"entropy": 0.036202426243107766,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.068776782602072,
|
|
"epoch": 0.0053125,
|
|
"step": 204
|
|
},
|
|
{
|
|
"loss": 0.27430152893066406,
|
|
"grad_norm": 1.8133119344711304,
|
|
"learning_rate": 1.6551724137931034e-07,
|
|
"num_tokens": 2184242.0,
|
|
"completions/mean_length": 845.125,
|
|
"completions/min_length": 233.0,
|
|
"completions/max_length": 1001.0,
|
|
"completions/clipped_ratio": 0.875,
|
|
"completions/mean_terminated_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"completions/max_terminated_length": 233.0,
|
|
"tools/call_frequency": 18.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.19895833730697632,
|
|
"rewards/reward_func/std": 0.23545750975608826,
|
|
"reward": 0.19895833730697632,
|
|
"reward_std": 0.23545750975608826,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001214715070091188,
|
|
"sampling/sampling_logp_difference/max": 0.4437544345855713,
|
|
"sampling/importance_sampling_ratio/min": 0.5729287266731262,
|
|
"sampling/importance_sampling_ratio/mean": 1.1272997856140137,
|
|
"sampling/importance_sampling_ratio/max": 2.4519150257110596,
|
|
"kl": 0.00073474441342114,
|
|
"entropy": 0.025819554488407448,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.59445869922638,
|
|
"epoch": 0.005338541666666667,
|
|
"step": 205
|
|
},
|
|
{
|
|
"loss": 0.2896766662597656,
|
|
"grad_norm": 1.695332646369934,
|
|
"learning_rate": 1.6379310344827585e-07,
|
|
"num_tokens": 2196391.0,
|
|
"completions/mean_length": 833.75,
|
|
"completions/min_length": 241.0,
|
|
"completions/max_length": 953.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 597.0,
|
|
"completions/min_terminated_length": 241.0,
|
|
"completions/max_terminated_length": 953.0,
|
|
"tools/call_frequency": 19.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21979168057441711,
|
|
"rewards/reward_func/std": 0.3144010007381439,
|
|
"reward": 0.21979168057441711,
|
|
"reward_std": 0.31440097093582153,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0010576180648058653,
|
|
"sampling/sampling_logp_difference/max": 0.5415244102478027,
|
|
"sampling/importance_sampling_ratio/min": 0.6405860185623169,
|
|
"sampling/importance_sampling_ratio/mean": 1.034003734588623,
|
|
"sampling/importance_sampling_ratio/max": 1.8133410215377808,
|
|
"kl": 0.0005409695093021583,
|
|
"entropy": 0.016931025311350822,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.811523055657744,
|
|
"epoch": 0.005364583333333333,
|
|
"step": 206
|
|
},
|
|
{
|
|
"loss": 0.10460381954908371,
|
|
"grad_norm": 1.384050726890564,
|
|
"learning_rate": 1.6206896551724136e-07,
|
|
"num_tokens": 2206914.0,
|
|
"completions/mean_length": 630.5,
|
|
"completions/min_length": 75.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 461.20001220703125,
|
|
"completions/min_terminated_length": 75.0,
|
|
"completions/max_terminated_length": 913.0,
|
|
"tools/call_frequency": 14.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13333334028720856,
|
|
"rewards/reward_func/std": 0.2768874764442444,
|
|
"reward": 0.13333334028720856,
|
|
"reward_std": 0.2768874764442444,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015999020542949438,
|
|
"sampling/sampling_logp_difference/max": 0.37247681617736816,
|
|
"sampling/importance_sampling_ratio/min": 0.5318619608879089,
|
|
"sampling/importance_sampling_ratio/mean": 0.911178469657898,
|
|
"sampling/importance_sampling_ratio/max": 1.2100470066070557,
|
|
"kl": 0.0006382451774697984,
|
|
"entropy": 0.02990632818546146,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.862965930253267,
|
|
"epoch": 0.005390625,
|
|
"step": 207
|
|
},
|
|
{
|
|
"loss": 0.7030704617500305,
|
|
"grad_norm": 4.527238368988037,
|
|
"learning_rate": 1.6034482758620686e-07,
|
|
"num_tokens": 2216328.0,
|
|
"completions/mean_length": 491.625,
|
|
"completions/min_length": 179.0,
|
|
"completions/max_length": 937.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 348.16668701171875,
|
|
"completions/min_terminated_length": 179.0,
|
|
"completions/max_terminated_length": 907.0,
|
|
"tools/call_frequency": 11.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.40312498807907104,
|
|
"rewards/reward_func/std": 0.32753702998161316,
|
|
"reward": 0.40312498807907104,
|
|
"reward_std": 0.32753702998161316,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.00327586242929101,
|
|
"sampling/sampling_logp_difference/max": 0.6249992847442627,
|
|
"sampling/importance_sampling_ratio/min": 0.46198996901512146,
|
|
"sampling/importance_sampling_ratio/mean": 0.941753625869751,
|
|
"sampling/importance_sampling_ratio/max": 2.2261922359466553,
|
|
"kl": 0.003055894927456393,
|
|
"entropy": 0.051408989704214036,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.459137689322233,
|
|
"epoch": 0.005416666666666667,
|
|
"step": 208
|
|
},
|
|
{
|
|
"loss": -0.08295954763889313,
|
|
"grad_norm": 1.8475444316864014,
|
|
"learning_rate": 1.5862068965517243e-07,
|
|
"num_tokens": 2228272.0,
|
|
"completions/mean_length": 808.0,
|
|
"completions/min_length": 141.0,
|
|
"completions/max_length": 933.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 714.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"completions/max_terminated_length": 918.0,
|
|
"tools/call_frequency": 19.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1059027835726738,
|
|
"rewards/reward_func/std": 0.22493599355220795,
|
|
"reward": 0.1059027835726738,
|
|
"reward_std": 0.22493599355220795,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014432737370952964,
|
|
"sampling/sampling_logp_difference/max": 0.6759881973266602,
|
|
"sampling/importance_sampling_ratio/min": 0.1562241017818451,
|
|
"sampling/importance_sampling_ratio/mean": 0.8241316080093384,
|
|
"sampling/importance_sampling_ratio/max": 1.1566932201385498,
|
|
"kl": 0.0004496565206864034,
|
|
"entropy": 0.022004504455253482,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.567789118736982,
|
|
"epoch": 0.005442708333333333,
|
|
"step": 209
|
|
},
|
|
{
|
|
"loss": -0.08953166007995605,
|
|
"grad_norm": 1.6254692077636719,
|
|
"learning_rate": 1.5689655172413793e-07,
|
|
"num_tokens": 2238304.0,
|
|
"completions/mean_length": 570.375,
|
|
"completions/min_length": 177.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 527.4285888671875,
|
|
"completions/min_terminated_length": 177.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 13.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2901041507720947,
|
|
"rewards/reward_func/std": 0.4220599830150604,
|
|
"reward": 0.2901041507720947,
|
|
"reward_std": 0.42205992341041565,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013964275131002069,
|
|
"sampling/sampling_logp_difference/max": 0.2809281349182129,
|
|
"sampling/importance_sampling_ratio/min": 0.6535372138023376,
|
|
"sampling/importance_sampling_ratio/mean": 0.9015812277793884,
|
|
"sampling/importance_sampling_ratio/max": 1.403028130531311,
|
|
"kl": 0.0005543840770769748,
|
|
"entropy": 0.04289804334985092,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.1607213485986,
|
|
"epoch": 0.00546875,
|
|
"step": 210
|
|
},
|
|
{
|
|
"loss": 0.027689699083566666,
|
|
"grad_norm": 1.6016696691513062,
|
|
"learning_rate": 1.5517241379310344e-07,
|
|
"num_tokens": 2248335.0,
|
|
"completions/mean_length": 568.0,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 967.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 369.6000061035156,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 967.0,
|
|
"tools/call_frequency": 13.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1275094747543335,
|
|
"rewards/reward_func/std": 0.2515515685081482,
|
|
"reward": 0.1275094747543335,
|
|
"reward_std": 0.2515515685081482,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0023312156554311514,
|
|
"sampling/sampling_logp_difference/max": 0.8754768967628479,
|
|
"sampling/importance_sampling_ratio/min": 0.30176806449890137,
|
|
"sampling/importance_sampling_ratio/mean": 0.9003896713256836,
|
|
"sampling/importance_sampling_ratio/max": 1.9475643634796143,
|
|
"kl": 0.0009815127177716931,
|
|
"entropy": 0.03295190801145509,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.577035777270794,
|
|
"epoch": 0.005494791666666667,
|
|
"step": 211
|
|
},
|
|
{
|
|
"loss": 0.13042065501213074,
|
|
"grad_norm": 3.1962993144989014,
|
|
"learning_rate": 1.5344827586206895e-07,
|
|
"num_tokens": 2258391.0,
|
|
"completions/mean_length": 571.875,
|
|
"completions/min_length": 164.0,
|
|
"completions/max_length": 949.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 520.857177734375,
|
|
"completions/min_terminated_length": 164.0,
|
|
"completions/max_terminated_length": 949.0,
|
|
"tools/call_frequency": 13.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.09583333879709244,
|
|
"rewards/reward_func/std": 0.24230213463306427,
|
|
"reward": 0.09583333879709244,
|
|
"reward_std": 0.24230211973190308,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016082286601886153,
|
|
"sampling/sampling_logp_difference/max": 0.5120277404785156,
|
|
"sampling/importance_sampling_ratio/min": 0.7312378287315369,
|
|
"sampling/importance_sampling_ratio/mean": 0.9514797329902649,
|
|
"sampling/importance_sampling_ratio/max": 1.499810814857483,
|
|
"kl": 0.002168090636132547,
|
|
"entropy": 0.04131688643246889,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.941068205982447,
|
|
"epoch": 0.005520833333333333,
|
|
"step": 212
|
|
},
|
|
{
|
|
"loss": 0.056521911174058914,
|
|
"grad_norm": 3.039984941482544,
|
|
"learning_rate": 1.5172413793103449e-07,
|
|
"num_tokens": 2269927.0,
|
|
"completions/mean_length": 756.25,
|
|
"completions/min_length": 263.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 473.3333435058594,
|
|
"completions/min_terminated_length": 263.0,
|
|
"completions/max_terminated_length": 853.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.24156251549720764,
|
|
"rewards/reward_func/std": 0.24112024903297424,
|
|
"reward": 0.24156251549720764,
|
|
"reward_std": 0.24112023413181305,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013848593225702643,
|
|
"sampling/sampling_logp_difference/max": 0.4956502914428711,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.9454419016838074,
|
|
"sampling/importance_sampling_ratio/max": 1.8225350379943848,
|
|
"kl": 0.0010036459962066147,
|
|
"entropy": 0.027799808885902166,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.541795782744884,
|
|
"epoch": 0.005546875,
|
|
"step": 213
|
|
},
|
|
{
|
|
"loss": -0.32676947116851807,
|
|
"grad_norm": 1.3860901594161987,
|
|
"learning_rate": 1.5e-07,
|
|
"num_tokens": 2281687.0,
|
|
"completions/mean_length": 783.875,
|
|
"completions/min_length": 244.0,
|
|
"completions/max_length": 949.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 701.2000122070312,
|
|
"completions/min_terminated_length": 244.0,
|
|
"completions/max_terminated_length": 930.0,
|
|
"tools/call_frequency": 18.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0012006998294964433,
|
|
"sampling/sampling_logp_difference/max": 0.26267847418785095,
|
|
"sampling/importance_sampling_ratio/min": 0.8224104642868042,
|
|
"sampling/importance_sampling_ratio/mean": 1.2633342742919922,
|
|
"sampling/importance_sampling_ratio/max": 1.753097414970398,
|
|
"kl": 0.00034639773730305023,
|
|
"entropy": 0.021867337520234287,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.518362207338214,
|
|
"epoch": 0.005572916666666667,
|
|
"step": 214
|
|
},
|
|
{
|
|
"loss": -0.05608800798654556,
|
|
"grad_norm": 1.037009596824646,
|
|
"learning_rate": 1.482758620689655e-07,
|
|
"num_tokens": 2291415.0,
|
|
"completions/mean_length": 530.125,
|
|
"completions/min_length": 44.0,
|
|
"completions/max_length": 978.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 388.8333435058594,
|
|
"completions/min_terminated_length": 44.0,
|
|
"completions/max_terminated_length": 938.0,
|
|
"tools/call_frequency": 11.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06979166716337204,
|
|
"rewards/reward_func/std": 0.30750977993011475,
|
|
"reward": 0.06979166716337204,
|
|
"reward_std": 0.30750975012779236,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001221520360559225,
|
|
"sampling/sampling_logp_difference/max": 0.7077305316925049,
|
|
"sampling/importance_sampling_ratio/min": 0.3418029546737671,
|
|
"sampling/importance_sampling_ratio/mean": 0.9600114226341248,
|
|
"sampling/importance_sampling_ratio/max": 1.4197125434875488,
|
|
"kl": 0.000688673042532173,
|
|
"entropy": 0.024413107079453766,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.649919990450144,
|
|
"epoch": 0.005598958333333333,
|
|
"step": 215
|
|
},
|
|
{
|
|
"loss": -0.0006911568343639374,
|
|
"grad_norm": 1.514074444770813,
|
|
"learning_rate": 1.46551724137931e-07,
|
|
"num_tokens": 2304196.0,
|
|
"completions/mean_length": 913.0,
|
|
"completions/min_length": 875.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 922.5,
|
|
"completions/min_terminated_length": 917.0,
|
|
"completions/max_terminated_length": 928.0,
|
|
"tools/call_frequency": 21.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13854166865348816,
|
|
"rewards/reward_func/std": 0.17621107399463654,
|
|
"reward": 0.13854166865348816,
|
|
"reward_std": 0.17621107399463654,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0008461082470603287,
|
|
"sampling/sampling_logp_difference/max": 0.4399830102920532,
|
|
"sampling/importance_sampling_ratio/min": 0.36119818687438965,
|
|
"sampling/importance_sampling_ratio/mean": 0.8764487504959106,
|
|
"sampling/importance_sampling_ratio/max": 1.4674574136734009,
|
|
"kl": 0.0002169548984056746,
|
|
"entropy": 0.0077959264745004475,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.231594074517488,
|
|
"epoch": 0.005625,
|
|
"step": 216
|
|
},
|
|
{
|
|
"loss": 0.0492577888071537,
|
|
"grad_norm": 2.669909715652466,
|
|
"learning_rate": 1.4482758620689654e-07,
|
|
"num_tokens": 2315732.0,
|
|
"completions/mean_length": 756.375,
|
|
"completions/min_length": 281.0,
|
|
"completions/max_length": 942.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 599.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"completions/max_terminated_length": 929.0,
|
|
"tools/call_frequency": 17.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2954450845718384,
|
|
"rewards/reward_func/std": 0.2404196858406067,
|
|
"reward": 0.2954450845718384,
|
|
"reward_std": 0.2404196560382843,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001257044030353427,
|
|
"sampling/sampling_logp_difference/max": 0.4844179153442383,
|
|
"sampling/importance_sampling_ratio/min": 0.500093936920166,
|
|
"sampling/importance_sampling_ratio/mean": 1.2580807209014893,
|
|
"sampling/importance_sampling_ratio/max": 2.6382806301116943,
|
|
"kl": 0.0009124866965066758,
|
|
"entropy": 0.03666805766988546,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.577289002016187,
|
|
"epoch": 0.005651041666666667,
|
|
"step": 217
|
|
},
|
|
{
|
|
"loss": -0.4558699429035187,
|
|
"grad_norm": 6.115187168121338,
|
|
"learning_rate": 1.4310344827586208e-07,
|
|
"num_tokens": 2326500.0,
|
|
"completions/mean_length": 661.125,
|
|
"completions/min_length": 183.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 237.33334350585938,
|
|
"completions/min_terminated_length": 183.0,
|
|
"completions/max_terminated_length": 273.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16354167461395264,
|
|
"rewards/reward_func/std": 0.2744741439819336,
|
|
"reward": 0.16354167461395264,
|
|
"reward_std": 0.2744741439819336,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002052918542176485,
|
|
"sampling/sampling_logp_difference/max": 0.6983792781829834,
|
|
"sampling/importance_sampling_ratio/min": 0.4170539975166321,
|
|
"sampling/importance_sampling_ratio/mean": 1.2802517414093018,
|
|
"sampling/importance_sampling_ratio/max": 2.415220260620117,
|
|
"kl": 0.001042486609549087,
|
|
"entropy": 0.04635164380306378,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.400666020810604,
|
|
"epoch": 0.0056770833333333335,
|
|
"step": 218
|
|
},
|
|
{
|
|
"loss": 0.2165909856557846,
|
|
"grad_norm": 2.9244587421417236,
|
|
"learning_rate": 1.413793103448276e-07,
|
|
"num_tokens": 2337263.0,
|
|
"completions/mean_length": 660.125,
|
|
"completions/min_length": 159.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 395.5,
|
|
"completions/min_terminated_length": 159.0,
|
|
"completions/max_terminated_length": 890.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06406249850988388,
|
|
"rewards/reward_func/std": 0.17467285692691803,
|
|
"reward": 0.06406249850988388,
|
|
"reward_std": 0.17467284202575684,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022286821622401476,
|
|
"sampling/sampling_logp_difference/max": 0.6936872005462646,
|
|
"sampling/importance_sampling_ratio/min": 0.276846319437027,
|
|
"sampling/importance_sampling_ratio/mean": 1.3623533248901367,
|
|
"sampling/importance_sampling_ratio/max": 2.6599323749542236,
|
|
"kl": 0.0007714900384598877,
|
|
"entropy": 0.04507393803214654,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.321143360808492,
|
|
"epoch": 0.005703125,
|
|
"step": 219
|
|
},
|
|
{
|
|
"loss": 0.025513142347335815,
|
|
"grad_norm": 1.9312078952789307,
|
|
"learning_rate": 1.396551724137931e-07,
|
|
"num_tokens": 2348010.0,
|
|
"completions/mean_length": 658.625,
|
|
"completions/min_length": 221.0,
|
|
"completions/max_length": 911.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 425.25,
|
|
"completions/min_terminated_length": 221.0,
|
|
"completions/max_terminated_length": 853.0,
|
|
"tools/call_frequency": 16.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.41644346714019775,
|
|
"rewards/reward_func/std": 0.3638288378715515,
|
|
"reward": 0.41644346714019775,
|
|
"reward_std": 0.3638288676738739,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002645940752699971,
|
|
"sampling/sampling_logp_difference/max": 1.0250787734985352,
|
|
"sampling/importance_sampling_ratio/min": 0.25876927375793457,
|
|
"sampling/importance_sampling_ratio/mean": 0.8641370534896851,
|
|
"sampling/importance_sampling_ratio/max": 1.7285879850387573,
|
|
"kl": 0.001306411459154333,
|
|
"entropy": 0.03776056398055516,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.09180523455143,
|
|
"epoch": 0.005729166666666666,
|
|
"step": 220
|
|
},
|
|
{
|
|
"loss": -0.07750144600868225,
|
|
"grad_norm": 3.3112220764160156,
|
|
"learning_rate": 1.379310344827586e-07,
|
|
"num_tokens": 2359410.0,
|
|
"completions/mean_length": 740.0,
|
|
"completions/min_length": 228.0,
|
|
"completions/max_length": 987.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 584.75,
|
|
"completions/min_terminated_length": 228.0,
|
|
"completions/max_terminated_length": 987.0,
|
|
"tools/call_frequency": 17.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.08437499403953552,
|
|
"rewards/reward_func/std": 0.17975056171417236,
|
|
"reward": 0.08437499403953552,
|
|
"reward_std": 0.17975056171417236,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0021046106703579426,
|
|
"sampling/sampling_logp_difference/max": 0.9922618865966797,
|
|
"sampling/importance_sampling_ratio/min": 0.2815137505531311,
|
|
"sampling/importance_sampling_ratio/mean": 1.1053941249847412,
|
|
"sampling/importance_sampling_ratio/max": 1.8996332883834839,
|
|
"kl": 0.0010763150939965271,
|
|
"entropy": 0.03213960549328476,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.9773825481534,
|
|
"epoch": 0.0057552083333333335,
|
|
"step": 221
|
|
},
|
|
{
|
|
"loss": -0.009444564580917358,
|
|
"grad_norm": 3.3041176795959473,
|
|
"learning_rate": 1.3620689655172414e-07,
|
|
"num_tokens": 2370202.0,
|
|
"completions/mean_length": 664.375,
|
|
"completions/min_length": 175.0,
|
|
"completions/max_length": 972.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 505.0,
|
|
"completions/min_terminated_length": 175.0,
|
|
"completions/max_terminated_length": 938.0,
|
|
"tools/call_frequency": 14.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.11875000596046448,
|
|
"rewards/reward_func/std": 0.1939946711063385,
|
|
"reward": 0.11875000596046448,
|
|
"reward_std": 0.1939946711063385,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019973055459558964,
|
|
"sampling/sampling_logp_difference/max": 0.8774774074554443,
|
|
"sampling/importance_sampling_ratio/min": 0.36275383830070496,
|
|
"sampling/importance_sampling_ratio/mean": 1.0513482093811035,
|
|
"sampling/importance_sampling_ratio/max": 2.2109670639038086,
|
|
"kl": 0.0006336443166219397,
|
|
"entropy": 0.03571362287038937,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.718975268304348,
|
|
"epoch": 0.00578125,
|
|
"step": 222
|
|
},
|
|
{
|
|
"loss": 0.18783168494701385,
|
|
"grad_norm": 3.571530818939209,
|
|
"learning_rate": 1.3448275862068965e-07,
|
|
"num_tokens": 2379737.0,
|
|
"completions/mean_length": 507.0,
|
|
"completions/min_length": 231.0,
|
|
"completions/max_length": 949.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 363.16668701171875,
|
|
"completions/min_terminated_length": 231.0,
|
|
"completions/max_terminated_length": 872.0,
|
|
"tools/call_frequency": 12.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3682291507720947,
|
|
"rewards/reward_func/std": 0.3328414261341095,
|
|
"reward": 0.3682291507720947,
|
|
"reward_std": 0.3328414261341095,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0036609170492738485,
|
|
"sampling/sampling_logp_difference/max": 0.773353099822998,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.7013465762138367,
|
|
"sampling/importance_sampling_ratio/max": 1.2049524784088135,
|
|
"kl": 0.0020519827576208627,
|
|
"entropy": 0.06061874941224232,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.865189021453261,
|
|
"epoch": 0.005807291666666666,
|
|
"step": 223
|
|
},
|
|
{
|
|
"loss": -0.22294186055660248,
|
|
"grad_norm": 2.820693016052246,
|
|
"learning_rate": 1.3275862068965515e-07,
|
|
"num_tokens": 2389893.0,
|
|
"completions/mean_length": 582.875,
|
|
"completions/min_length": 144.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 232.25,
|
|
"completions/min_terminated_length": 144.0,
|
|
"completions/max_terminated_length": 330.0,
|
|
"tools/call_frequency": 13.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.015625,
|
|
"rewards/reward_func/std": 0.11254959553480148,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.11254958808422089,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002143474528566003,
|
|
"sampling/sampling_logp_difference/max": 0.4155219793319702,
|
|
"sampling/importance_sampling_ratio/min": 0.6273090839385986,
|
|
"sampling/importance_sampling_ratio/mean": 0.9074385166168213,
|
|
"sampling/importance_sampling_ratio/max": 1.1173546314239502,
|
|
"kl": 0.0009552283481752966,
|
|
"entropy": 0.03704297775402665,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.569462371990085,
|
|
"epoch": 0.005833333333333334,
|
|
"step": 224
|
|
},
|
|
{
|
|
"loss": 0.1635877788066864,
|
|
"grad_norm": 3.1200647354125977,
|
|
"learning_rate": 1.310344827586207e-07,
|
|
"num_tokens": 2399361.0,
|
|
"completions/mean_length": 499.0,
|
|
"completions/min_length": 175.0,
|
|
"completions/max_length": 912.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 259.0,
|
|
"completions/min_terminated_length": 175.0,
|
|
"completions/max_terminated_length": 334.0,
|
|
"tools/call_frequency": 12.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23802083730697632,
|
|
"rewards/reward_func/std": 0.3573591113090515,
|
|
"reward": 0.23802083730697632,
|
|
"reward_std": 0.3573591113090515,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003416451858356595,
|
|
"sampling/sampling_logp_difference/max": 0.8391715288162231,
|
|
"sampling/importance_sampling_ratio/min": 0.4584909975528717,
|
|
"sampling/importance_sampling_ratio/mean": 1.0887699127197266,
|
|
"sampling/importance_sampling_ratio/max": 2.414736747741699,
|
|
"kl": 0.0012592731237646149,
|
|
"entropy": 0.05699187744176015,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.784927282482386,
|
|
"epoch": 0.005859375,
|
|
"step": 225
|
|
},
|
|
{
|
|
"loss": 0.24695104360580444,
|
|
"grad_norm": 1.8188022375106812,
|
|
"learning_rate": 1.293103448275862e-07,
|
|
"num_tokens": 2409616.0,
|
|
"completions/mean_length": 596.875,
|
|
"completions/min_length": 237.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 408.8000183105469,
|
|
"completions/min_terminated_length": 237.0,
|
|
"completions/max_terminated_length": 929.0,
|
|
"tools/call_frequency": 13.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2720833420753479,
|
|
"rewards/reward_func/std": 0.29608824849128723,
|
|
"reward": 0.2720833420753479,
|
|
"reward_std": 0.2960882782936096,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002701138611882925,
|
|
"sampling/sampling_logp_difference/max": 1.3009891510009766,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.6907131671905518,
|
|
"sampling/importance_sampling_ratio/max": 0.9938694834709167,
|
|
"kl": 0.0019287753011667519,
|
|
"entropy": 0.05428809206932783,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.264590693637729,
|
|
"epoch": 0.005885416666666666,
|
|
"step": 226
|
|
},
|
|
{
|
|
"loss": 0.09087327122688293,
|
|
"grad_norm": 2.3367080688476562,
|
|
"learning_rate": 1.2758620689655173e-07,
|
|
"num_tokens": 2420433.0,
|
|
"completions/mean_length": 665.75,
|
|
"completions/min_length": 149.0,
|
|
"completions/max_length": 963.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 517.2000122070312,
|
|
"completions/min_terminated_length": 149.0,
|
|
"completions/max_terminated_length": 963.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.07877840846776962,
|
|
"rewards/reward_func/std": 0.2136470377445221,
|
|
"reward": 0.07877840846776962,
|
|
"reward_std": 0.2136470228433609,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016126023838296533,
|
|
"sampling/sampling_logp_difference/max": 0.6487810611724854,
|
|
"sampling/importance_sampling_ratio/min": 0.5477168560028076,
|
|
"sampling/importance_sampling_ratio/mean": 0.9244718551635742,
|
|
"sampling/importance_sampling_ratio/max": 1.5821164846420288,
|
|
"kl": 0.0007234385690253475,
|
|
"entropy": 0.031039555033203214,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.695401513949037,
|
|
"epoch": 0.005911458333333334,
|
|
"step": 227
|
|
},
|
|
{
|
|
"loss": 0.2623760402202606,
|
|
"grad_norm": 0.8531036972999573,
|
|
"learning_rate": 1.2586206896551724e-07,
|
|
"num_tokens": 2432090.0,
|
|
"completions/mean_length": 770.625,
|
|
"completions/min_length": 305.0,
|
|
"completions/max_length": 933.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 684.2000122070312,
|
|
"completions/min_terminated_length": 305.0,
|
|
"completions/max_terminated_length": 933.0,
|
|
"tools/call_frequency": 17.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.234375,
|
|
"rewards/reward_func/std": 0.2666550576686859,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.2666550278663635,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016370323719456792,
|
|
"sampling/sampling_logp_difference/max": 0.44652771949768066,
|
|
"sampling/importance_sampling_ratio/min": 0.20589512586593628,
|
|
"sampling/importance_sampling_ratio/mean": 0.7134807109832764,
|
|
"sampling/importance_sampling_ratio/max": 1.1239194869995117,
|
|
"kl": 0.0007723774654095905,
|
|
"entropy": 0.03348025312880054,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.717260397970676,
|
|
"epoch": 0.0059375,
|
|
"step": 228
|
|
},
|
|
{
|
|
"loss": 0.17576314508914948,
|
|
"grad_norm": 2.5869407653808594,
|
|
"learning_rate": 1.2413793103448275e-07,
|
|
"num_tokens": 2443625.0,
|
|
"completions/mean_length": 755.625,
|
|
"completions/min_length": 233.0,
|
|
"completions/max_length": 941.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 492.3333435058594,
|
|
"completions/min_terminated_length": 233.0,
|
|
"completions/max_terminated_length": 912.0,
|
|
"tools/call_frequency": 17.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1875,
|
|
"rewards/reward_func/std": 0.20485186576843262,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.20485186576843262,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019748560152947903,
|
|
"sampling/sampling_logp_difference/max": 1.0082687139511108,
|
|
"sampling/importance_sampling_ratio/min": 0.46375036239624023,
|
|
"sampling/importance_sampling_ratio/mean": 0.8684507012367249,
|
|
"sampling/importance_sampling_ratio/max": 1.3558887243270874,
|
|
"kl": 0.00048642021420164383,
|
|
"entropy": 0.030559567036107183,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.349721631035209,
|
|
"epoch": 0.0059635416666666665,
|
|
"step": 229
|
|
},
|
|
{
|
|
"loss": 0.4225098788738251,
|
|
"grad_norm": 1.762121558189392,
|
|
"learning_rate": 1.2241379310344826e-07,
|
|
"num_tokens": 2453802.0,
|
|
"completions/mean_length": 586.375,
|
|
"completions/min_length": 140.0,
|
|
"completions/max_length": 978.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 253.75,
|
|
"completions/min_terminated_length": 140.0,
|
|
"completions/max_terminated_length": 330.0,
|
|
"tools/call_frequency": 13.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.20364582538604736,
|
|
"rewards/reward_func/std": 0.2686813473701477,
|
|
"reward": 0.20364582538604736,
|
|
"reward_std": 0.2686813473701477,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003027789294719696,
|
|
"sampling/sampling_logp_difference/max": 0.901741623878479,
|
|
"sampling/importance_sampling_ratio/min": 0.06553680449724197,
|
|
"sampling/importance_sampling_ratio/mean": 0.7198530435562134,
|
|
"sampling/importance_sampling_ratio/max": 1.6110345125198364,
|
|
"kl": 0.0011238102847528353,
|
|
"entropy": 0.04927462735213339,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.700101539492607,
|
|
"epoch": 0.005989583333333334,
|
|
"step": 230
|
|
},
|
|
{
|
|
"loss": 0.3389125466346741,
|
|
"grad_norm": 5.249733924865723,
|
|
"learning_rate": 1.206896551724138e-07,
|
|
"num_tokens": 2463921.0,
|
|
"completions/mean_length": 579.75,
|
|
"completions/min_length": 169.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 470.8333435058594,
|
|
"completions/min_terminated_length": 169.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2100694477558136,
|
|
"rewards/reward_func/std": 0.28630149364471436,
|
|
"reward": 0.2100694477558136,
|
|
"reward_std": 0.28630149364471436,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002475165296345949,
|
|
"sampling/sampling_logp_difference/max": 0.6478128433227539,
|
|
"sampling/importance_sampling_ratio/min": 0.3486902713775635,
|
|
"sampling/importance_sampling_ratio/mean": 0.9353398084640503,
|
|
"sampling/importance_sampling_ratio/max": 1.9876244068145752,
|
|
"kl": 0.001924207260572075,
|
|
"entropy": 0.04689964454155415,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.03075765632093,
|
|
"epoch": 0.006015625,
|
|
"step": 231
|
|
},
|
|
{
|
|
"loss": 0.10969091951847076,
|
|
"grad_norm": 2.1705100536346436,
|
|
"learning_rate": 1.1896551724137931e-07,
|
|
"num_tokens": 2473979.0,
|
|
"completions/mean_length": 571.25,
|
|
"completions/min_length": 221.0,
|
|
"completions/max_length": 922.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 248.5,
|
|
"completions/min_terminated_length": 221.0,
|
|
"completions/max_terminated_length": 271.0,
|
|
"tools/call_frequency": 14.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17031250894069672,
|
|
"rewards/reward_func/std": 0.27401918172836304,
|
|
"reward": 0.17031250894069672,
|
|
"reward_std": 0.27401915192604065,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017383125377818942,
|
|
"sampling/sampling_logp_difference/max": 0.41143089532852173,
|
|
"sampling/importance_sampling_ratio/min": 0.42801231145858765,
|
|
"sampling/importance_sampling_ratio/mean": 0.9083375334739685,
|
|
"sampling/importance_sampling_ratio/max": 1.3876216411590576,
|
|
"kl": 0.0008976281596915214,
|
|
"entropy": 0.02796859308728017,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.958634555339813,
|
|
"epoch": 0.0060416666666666665,
|
|
"step": 232
|
|
},
|
|
{
|
|
"loss": -0.1477838158607483,
|
|
"grad_norm": 1.9781155586242676,
|
|
"learning_rate": 1.1724137931034482e-07,
|
|
"num_tokens": 2484724.0,
|
|
"completions/mean_length": 657.75,
|
|
"completions/min_length": 193.0,
|
|
"completions/max_length": 942.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 239.6666717529297,
|
|
"completions/min_terminated_length": 193.0,
|
|
"completions/max_terminated_length": 333.0,
|
|
"tools/call_frequency": 15.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.16316963732242584,
|
|
"rewards/reward_func/std": 0.30970242619514465,
|
|
"reward": 0.16316963732242584,
|
|
"reward_std": 0.30970239639282227,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001989222364500165,
|
|
"sampling/sampling_logp_difference/max": 0.4092292785644531,
|
|
"sampling/importance_sampling_ratio/min": 0.24774305522441864,
|
|
"sampling/importance_sampling_ratio/mean": 0.801170825958252,
|
|
"sampling/importance_sampling_ratio/max": 1.111074447631836,
|
|
"kl": 0.0006999265660851961,
|
|
"entropy": 0.03680746193276718,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.823119714856148,
|
|
"epoch": 0.006067708333333333,
|
|
"step": 233
|
|
},
|
|
{
|
|
"loss": -0.11681036651134491,
|
|
"grad_norm": 1.3626787662506104,
|
|
"learning_rate": 1.1551724137931033e-07,
|
|
"num_tokens": 2495409.0,
|
|
"completions/mean_length": 649.5,
|
|
"completions/min_length": 144.0,
|
|
"completions/max_length": 942.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 485.6000061035156,
|
|
"completions/min_terminated_length": 144.0,
|
|
"completions/max_terminated_length": 930.0,
|
|
"tools/call_frequency": 15.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05624999850988388,
|
|
"rewards/reward_func/std": 0.1237436980009079,
|
|
"reward": 0.05624999850988388,
|
|
"reward_std": 0.1237436905503273,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014957230305299163,
|
|
"sampling/sampling_logp_difference/max": 0.7639822363853455,
|
|
"sampling/importance_sampling_ratio/min": 0.30512765049934387,
|
|
"sampling/importance_sampling_ratio/mean": 0.8351577520370483,
|
|
"sampling/importance_sampling_ratio/max": 1.488519310951233,
|
|
"kl": 0.0004947063898725901,
|
|
"entropy": 0.02667384606320411,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.060222905129194,
|
|
"epoch": 0.00609375,
|
|
"step": 234
|
|
},
|
|
{
|
|
"loss": 0.015743223950266838,
|
|
"grad_norm": 2.1963679790496826,
|
|
"learning_rate": 1.1379310344827586e-07,
|
|
"num_tokens": 2505382.0,
|
|
"completions/mean_length": 561.375,
|
|
"completions/min_length": 170.0,
|
|
"completions/max_length": 942.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 342.3999938964844,
|
|
"completions/min_terminated_length": 170.0,
|
|
"completions/max_terminated_length": 913.0,
|
|
"tools/call_frequency": 13.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.07916666567325592,
|
|
"rewards/reward_func/std": 0.27454873919487,
|
|
"reward": 0.07916666567325592,
|
|
"reward_std": 0.2745487093925476,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019861096516251564,
|
|
"sampling/sampling_logp_difference/max": 0.4399799108505249,
|
|
"sampling/importance_sampling_ratio/min": 0.4819249212741852,
|
|
"sampling/importance_sampling_ratio/mean": 0.775201678276062,
|
|
"sampling/importance_sampling_ratio/max": 1.0341510772705078,
|
|
"kl": 0.0018405111140964436,
|
|
"entropy": 0.03506177477538586,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.160426128655672,
|
|
"epoch": 0.006119791666666667,
|
|
"step": 235
|
|
},
|
|
{
|
|
"loss": -0.34173041582107544,
|
|
"grad_norm": 1.5645779371261597,
|
|
"learning_rate": 1.1206896551724137e-07,
|
|
"num_tokens": 2517381.0,
|
|
"completions/mean_length": 814.75,
|
|
"completions/min_length": 168.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 547.0,
|
|
"completions/min_terminated_length": 168.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 20.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0008157703559845686,
|
|
"sampling/sampling_logp_difference/max": 0.4873208999633789,
|
|
"sampling/importance_sampling_ratio/min": 0.4836920201778412,
|
|
"sampling/importance_sampling_ratio/mean": 1.055680751800537,
|
|
"sampling/importance_sampling_ratio/max": 1.894108533859253,
|
|
"kl": 0.00019116300268251507,
|
|
"entropy": 0.013480905094183981,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.23903290182352,
|
|
"epoch": 0.006145833333333333,
|
|
"step": 236
|
|
},
|
|
{
|
|
"loss": -0.17320576310157776,
|
|
"grad_norm": 2.5685641765594482,
|
|
"learning_rate": 1.103448275862069e-07,
|
|
"num_tokens": 2529047.0,
|
|
"completions/mean_length": 773.375,
|
|
"completions/min_length": 193.0,
|
|
"completions/max_length": 959.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 665.0,
|
|
"completions/min_terminated_length": 193.0,
|
|
"completions/max_terminated_length": 959.0,
|
|
"tools/call_frequency": 19.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.0729166716337204,
|
|
"rewards/reward_func/std": 0.13596728444099426,
|
|
"reward": 0.0729166716337204,
|
|
"reward_std": 0.13596728444099426,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0012909258948639035,
|
|
"sampling/sampling_logp_difference/max": 0.5969448089599609,
|
|
"sampling/importance_sampling_ratio/min": 0.7327394485473633,
|
|
"sampling/importance_sampling_ratio/mean": 0.9812209606170654,
|
|
"sampling/importance_sampling_ratio/max": 1.4440574645996094,
|
|
"kl": 0.0013069981250737328,
|
|
"entropy": 0.014995089848525822,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.0440405420959,
|
|
"epoch": 0.006171875,
|
|
"step": 237
|
|
},
|
|
{
|
|
"loss": -0.4220482409000397,
|
|
"grad_norm": 1.4120210409164429,
|
|
"learning_rate": 1.086206896551724e-07,
|
|
"num_tokens": 2540424.0,
|
|
"completions/mean_length": 736.875,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 944.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 431.66668701171875,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 914.0,
|
|
"tools/call_frequency": 17.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.012500000186264515,
|
|
"rewards/reward_func/std": 0.06943651288747787,
|
|
"reward": 0.012500000186264515,
|
|
"reward_std": 0.06943650543689728,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0009736094507388771,
|
|
"sampling/sampling_logp_difference/max": 0.27872633934020996,
|
|
"sampling/importance_sampling_ratio/min": 0.6991724967956543,
|
|
"sampling/importance_sampling_ratio/mean": 1.019271731376648,
|
|
"sampling/importance_sampling_ratio/max": 1.5726211071014404,
|
|
"kl": 0.0004879603889094142,
|
|
"entropy": 0.020592384331393987,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.751837013289332,
|
|
"epoch": 0.006197916666666667,
|
|
"step": 238
|
|
},
|
|
{
|
|
"loss": 0.09794958680868149,
|
|
"grad_norm": 1.3610752820968628,
|
|
"learning_rate": 1.0689655172413794e-07,
|
|
"num_tokens": 2550608.0,
|
|
"completions/mean_length": 588.125,
|
|
"completions/min_length": 221.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 500.3333435058594,
|
|
"completions/min_terminated_length": 221.0,
|
|
"completions/max_terminated_length": 921.0,
|
|
"tools/call_frequency": 14.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2854166626930237,
|
|
"rewards/reward_func/std": 0.3148050010204315,
|
|
"reward": 0.2854166626930237,
|
|
"reward_std": 0.3148050010204315,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002714649075642228,
|
|
"sampling/sampling_logp_difference/max": 0.6081104278564453,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.7740776538848877,
|
|
"sampling/importance_sampling_ratio/max": 2.679922580718994,
|
|
"kl": 0.0013353467784327222,
|
|
"entropy": 0.041203959146514535,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.328858561813831,
|
|
"epoch": 0.006223958333333333,
|
|
"step": 239
|
|
},
|
|
{
|
|
"loss": -0.23006215691566467,
|
|
"grad_norm": 2.4470808506011963,
|
|
"learning_rate": 1.0517241379310344e-07,
|
|
"num_tokens": 2562441.0,
|
|
"completions/mean_length": 794.5,
|
|
"completions/min_length": 330.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 617.3333740234375,
|
|
"completions/min_terminated_length": 330.0,
|
|
"completions/max_terminated_length": 930.0,
|
|
"tools/call_frequency": 18.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.24288195371627808,
|
|
"rewards/reward_func/std": 0.28983309864997864,
|
|
"reward": 0.24288195371627808,
|
|
"reward_std": 0.28983306884765625,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001453027711249888,
|
|
"sampling/sampling_logp_difference/max": 0.512680172920227,
|
|
"sampling/importance_sampling_ratio/min": 0.22651466727256775,
|
|
"sampling/importance_sampling_ratio/mean": 0.8187382817268372,
|
|
"sampling/importance_sampling_ratio/max": 1.3410955667495728,
|
|
"kl": 0.0009412031383817521,
|
|
"entropy": 0.023144229082390666,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.449569009244442,
|
|
"epoch": 0.00625,
|
|
"step": 240
|
|
},
|
|
{
|
|
"loss": -0.021215975284576416,
|
|
"grad_norm": 1.6600942611694336,
|
|
"learning_rate": 1.0344827586206897e-07,
|
|
"num_tokens": 2573142.0,
|
|
"completions/mean_length": 652.0,
|
|
"completions/min_length": 148.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 227.6666717529297,
|
|
"completions/min_terminated_length": 148.0,
|
|
"completions/max_terminated_length": 289.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2725694477558136,
|
|
"rewards/reward_func/std": 0.2978000044822693,
|
|
"reward": 0.2725694477558136,
|
|
"reward_std": 0.2978000044822693,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017026587156578898,
|
|
"sampling/sampling_logp_difference/max": 0.4437239170074463,
|
|
"sampling/importance_sampling_ratio/min": 0.6040502786636353,
|
|
"sampling/importance_sampling_ratio/mean": 1.0377980470657349,
|
|
"sampling/importance_sampling_ratio/max": 1.631766438484192,
|
|
"kl": 0.0009313494188063487,
|
|
"entropy": 0.038280706328805536,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.727393535897136,
|
|
"epoch": 0.006276041666666667,
|
|
"step": 241
|
|
},
|
|
{
|
|
"loss": -0.3134765625,
|
|
"grad_norm": 1.1355687379837036,
|
|
"learning_rate": 1.0172413793103447e-07,
|
|
"num_tokens": 2584542.0,
|
|
"completions/mean_length": 738.875,
|
|
"completions/min_length": 150.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 415.0,
|
|
"completions/min_terminated_length": 150.0,
|
|
"completions/max_terminated_length": 922.0,
|
|
"tools/call_frequency": 16.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06562499701976776,
|
|
"rewards/reward_func/std": 0.17875438928604126,
|
|
"reward": 0.06562499701976776,
|
|
"reward_std": 0.17875437438488007,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0011290323454886675,
|
|
"sampling/sampling_logp_difference/max": 0.30381929874420166,
|
|
"sampling/importance_sampling_ratio/min": 0.4889325201511383,
|
|
"sampling/importance_sampling_ratio/mean": 0.9566811323165894,
|
|
"sampling/importance_sampling_ratio/max": 1.3045167922973633,
|
|
"kl": 0.00046188371993594046,
|
|
"entropy": 0.020167565904557705,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.597209522500634,
|
|
"epoch": 0.006302083333333333,
|
|
"step": 242
|
|
},
|
|
{
|
|
"loss": -0.020207688212394714,
|
|
"grad_norm": 1.931732416152954,
|
|
"learning_rate": 1e-07,
|
|
"num_tokens": 2594588.0,
|
|
"completions/mean_length": 570.5,
|
|
"completions/min_length": 154.0,
|
|
"completions/max_length": 924.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 227.0,
|
|
"completions/min_terminated_length": 154.0,
|
|
"completions/max_terminated_length": 309.0,
|
|
"tools/call_frequency": 13.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05937499552965164,
|
|
"rewards/reward_func/std": 0.22117280960083008,
|
|
"reward": 0.05937499552965164,
|
|
"reward_std": 0.22117280960083008,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020563669968396425,
|
|
"sampling/sampling_logp_difference/max": 0.46228551864624023,
|
|
"sampling/importance_sampling_ratio/min": 0.3523009717464447,
|
|
"sampling/importance_sampling_ratio/mean": 0.7563523054122925,
|
|
"sampling/importance_sampling_ratio/max": 1.1243330240249634,
|
|
"kl": 0.0006057715772840311,
|
|
"entropy": 0.038295404287055135,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.851736659184098,
|
|
"epoch": 0.006328125,
|
|
"step": 243
|
|
},
|
|
{
|
|
"loss": -0.1939881443977356,
|
|
"grad_norm": 2.9497697353363037,
|
|
"learning_rate": 9.827586206896552e-08,
|
|
"num_tokens": 2604516.0,
|
|
"completions/mean_length": 555.625,
|
|
"completions/min_length": 102.0,
|
|
"completions/max_length": 958.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 192.0,
|
|
"completions/min_terminated_length": 102.0,
|
|
"completions/max_terminated_length": 259.0,
|
|
"tools/call_frequency": 12.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.08697916567325592,
|
|
"rewards/reward_func/std": 0.23104146122932434,
|
|
"reward": 0.08697916567325592,
|
|
"reward_std": 0.23104146122932434,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0023398431949317455,
|
|
"sampling/sampling_logp_difference/max": 1.1150586605072021,
|
|
"sampling/importance_sampling_ratio/min": 0.23312920331954956,
|
|
"sampling/importance_sampling_ratio/mean": 0.9304143190383911,
|
|
"sampling/importance_sampling_ratio/max": 1.6181161403656006,
|
|
"kl": 0.0011754528632081929,
|
|
"entropy": 0.03973157057771459,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.032157097011805,
|
|
"epoch": 0.006354166666666667,
|
|
"step": 244
|
|
},
|
|
{
|
|
"loss": -0.24688518047332764,
|
|
"grad_norm": 1.81999671459198,
|
|
"learning_rate": 9.655172413793103e-08,
|
|
"num_tokens": 2616650.0,
|
|
"completions/mean_length": 831.875,
|
|
"completions/min_length": 177.0,
|
|
"completions/max_length": 1003.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 673.6666870117188,
|
|
"completions/min_terminated_length": 177.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 19.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0010624408023431897,
|
|
"sampling/sampling_logp_difference/max": 0.2809312343597412,
|
|
"sampling/importance_sampling_ratio/min": 0.5365629196166992,
|
|
"sampling/importance_sampling_ratio/mean": 0.9433606863021851,
|
|
"sampling/importance_sampling_ratio/max": 1.4678759574890137,
|
|
"kl": 0.00021945680327917216,
|
|
"entropy": 0.015761735528940335,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.017802283167839,
|
|
"epoch": 0.006380208333333333,
|
|
"step": 245
|
|
},
|
|
{
|
|
"loss": 0.12044897675514221,
|
|
"grad_norm": 1.4615535736083984,
|
|
"learning_rate": 9.482758620689655e-08,
|
|
"num_tokens": 2628256.0,
|
|
"completions/mean_length": 765.0,
|
|
"completions/min_length": 243.0,
|
|
"completions/max_length": 946.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 594.25,
|
|
"completions/min_terminated_length": 243.0,
|
|
"completions/max_terminated_length": 930.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06041666865348816,
|
|
"rewards/reward_func/std": 0.10424996167421341,
|
|
"reward": 0.06041666865348816,
|
|
"reward_std": 0.10424996167421341,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019089553970843554,
|
|
"sampling/sampling_logp_difference/max": 1.6990818977355957,
|
|
"sampling/importance_sampling_ratio/min": 0.2568036913871765,
|
|
"sampling/importance_sampling_ratio/mean": 0.8895500898361206,
|
|
"sampling/importance_sampling_ratio/max": 1.2761133909225464,
|
|
"kl": 0.00042345532892795745,
|
|
"entropy": 0.027600726461969316,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.04678756557405,
|
|
"epoch": 0.00640625,
|
|
"step": 246
|
|
},
|
|
{
|
|
"loss": -0.11552298069000244,
|
|
"grad_norm": 2.193773031234741,
|
|
"learning_rate": 9.310344827586207e-08,
|
|
"num_tokens": 2638364.0,
|
|
"completions/mean_length": 577.75,
|
|
"completions/min_length": 222.0,
|
|
"completions/max_length": 915.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 387.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"completions/max_terminated_length": 847.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.39268356561660767,
|
|
"rewards/reward_func/std": 0.32454654574394226,
|
|
"reward": 0.39268356561660767,
|
|
"reward_std": 0.32454654574394226,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002838875399902463,
|
|
"sampling/sampling_logp_difference/max": 0.5075106620788574,
|
|
"sampling/importance_sampling_ratio/min": 0.4344515800476074,
|
|
"sampling/importance_sampling_ratio/mean": 0.930603563785553,
|
|
"sampling/importance_sampling_ratio/max": 1.4186346530914307,
|
|
"kl": 0.0011452651124272961,
|
|
"entropy": 0.05036524042952806,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.460650473833084,
|
|
"epoch": 0.006432291666666667,
|
|
"step": 247
|
|
},
|
|
{
|
|
"loss": 0.09233029931783676,
|
|
"grad_norm": 3.070572853088379,
|
|
"learning_rate": 9.137931034482759e-08,
|
|
"num_tokens": 2648365.0,
|
|
"completions/mean_length": 564.5,
|
|
"completions/min_length": 185.0,
|
|
"completions/max_length": 937.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 448.16668701171875,
|
|
"completions/min_terminated_length": 185.0,
|
|
"completions/max_terminated_length": 937.0,
|
|
"tools/call_frequency": 13.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13072916865348816,
|
|
"rewards/reward_func/std": 0.3315199017524719,
|
|
"reward": 0.13072916865348816,
|
|
"reward_std": 0.3315199017524719,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002198112430050969,
|
|
"sampling/sampling_logp_difference/max": 0.6983113288879395,
|
|
"sampling/importance_sampling_ratio/min": 0.6714800596237183,
|
|
"sampling/importance_sampling_ratio/mean": 1.497720718383789,
|
|
"sampling/importance_sampling_ratio/max": 2.1813416481018066,
|
|
"kl": 0.0012601845232893538,
|
|
"entropy": 0.03557961562182754,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.00173356384039,
|
|
"epoch": 0.006458333333333333,
|
|
"step": 248
|
|
},
|
|
{
|
|
"loss": 0.20256459712982178,
|
|
"grad_norm": 9.362682342529297,
|
|
"learning_rate": 8.96551724137931e-08,
|
|
"num_tokens": 2658504.0,
|
|
"completions/mean_length": 581.875,
|
|
"completions/min_length": 157.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 459.8333435058594,
|
|
"completions/min_terminated_length": 157.0,
|
|
"completions/max_terminated_length": 910.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.15156248211860657,
|
|
"rewards/reward_func/std": 0.30196067690849304,
|
|
"reward": 0.15156248211860657,
|
|
"reward_std": 0.30196067690849304,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0021101117599755526,
|
|
"sampling/sampling_logp_difference/max": 0.6983802318572998,
|
|
"sampling/importance_sampling_ratio/min": 0.32187071442604065,
|
|
"sampling/importance_sampling_ratio/mean": 0.739851713180542,
|
|
"sampling/importance_sampling_ratio/max": 1.6177717447280884,
|
|
"kl": 0.0014618789637097507,
|
|
"entropy": 0.03782705048797652,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.598702143877745,
|
|
"epoch": 0.006484375,
|
|
"step": 249
|
|
},
|
|
{
|
|
"loss": 0.41971254348754883,
|
|
"grad_norm": 2.333144426345825,
|
|
"learning_rate": 8.79310344827586e-08,
|
|
"num_tokens": 2667315.0,
|
|
"completions/mean_length": 415.375,
|
|
"completions/min_length": 182.0,
|
|
"completions/max_length": 917.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 249.0,
|
|
"completions/min_terminated_length": 182.0,
|
|
"completions/max_terminated_length": 346.0,
|
|
"tools/call_frequency": 9.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.20520833134651184,
|
|
"rewards/reward_func/std": 0.3083072006702423,
|
|
"reward": 0.20520833134651184,
|
|
"reward_std": 0.3083072006702423,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.004471456166356802,
|
|
"sampling/sampling_logp_difference/max": 0.4964871406555176,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8965224027633667,
|
|
"sampling/importance_sampling_ratio/max": 1.84627366065979,
|
|
"kl": 0.0018817327227225178,
|
|
"entropy": 0.05974789569154382,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.395800825208426,
|
|
"epoch": 0.006510416666666667,
|
|
"step": 250
|
|
},
|
|
{
|
|
"loss": 0.06133618950843811,
|
|
"grad_norm": 2.390864610671997,
|
|
"learning_rate": 8.620689655172414e-08,
|
|
"num_tokens": 2679421.0,
|
|
"completions/mean_length": 827.75,
|
|
"completions/min_length": 253.0,
|
|
"completions/max_length": 956.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 584.0,
|
|
"completions/min_terminated_length": 253.0,
|
|
"completions/max_terminated_length": 915.0,
|
|
"tools/call_frequency": 19.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.19971591234207153,
|
|
"rewards/reward_func/std": 0.232671856880188,
|
|
"reward": 0.19971591234207153,
|
|
"reward_std": 0.232671856880188,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014988395851105452,
|
|
"sampling/sampling_logp_difference/max": 0.4331512451171875,
|
|
"sampling/importance_sampling_ratio/min": 0.361944317817688,
|
|
"sampling/importance_sampling_ratio/mean": 0.9045666456222534,
|
|
"sampling/importance_sampling_ratio/max": 1.7648506164550781,
|
|
"kl": 0.0007280136114786728,
|
|
"entropy": 0.02289991726865992,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.835198134183884,
|
|
"epoch": 0.006536458333333333,
|
|
"step": 251
|
|
},
|
|
{
|
|
"loss": -0.14060615003108978,
|
|
"grad_norm": 1.6411021947860718,
|
|
"learning_rate": 8.448275862068965e-08,
|
|
"num_tokens": 2690105.0,
|
|
"completions/mean_length": 649.375,
|
|
"completions/min_length": 182.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 486.3999938964844,
|
|
"completions/min_terminated_length": 182.0,
|
|
"completions/max_terminated_length": 886.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.05937500670552254,
|
|
"rewards/reward_func/std": 0.1295166164636612,
|
|
"reward": 0.05937500670552254,
|
|
"reward_std": 0.1295166164636612,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018440443091094494,
|
|
"sampling/sampling_logp_difference/max": 0.4519460201263428,
|
|
"sampling/importance_sampling_ratio/min": 0.3875502347946167,
|
|
"sampling/importance_sampling_ratio/mean": 0.7990303039550781,
|
|
"sampling/importance_sampling_ratio/max": 1.1202245950698853,
|
|
"kl": 0.001108981852667057,
|
|
"entropy": 0.03489629703108221,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.890469036996365,
|
|
"epoch": 0.0065625,
|
|
"step": 252
|
|
},
|
|
{
|
|
"loss": 0.07681693136692047,
|
|
"grad_norm": 1.006963849067688,
|
|
"learning_rate": 8.275862068965517e-08,
|
|
"num_tokens": 2701572.0,
|
|
"completions/mean_length": 746.75,
|
|
"completions/min_length": 189.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 587.25,
|
|
"completions/min_terminated_length": 189.0,
|
|
"completions/max_terminated_length": 897.0,
|
|
"tools/call_frequency": 17.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13608631491661072,
|
|
"rewards/reward_func/std": 0.2127850353717804,
|
|
"reward": 0.13608631491661072,
|
|
"reward_std": 0.2127850353717804,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001649903366342187,
|
|
"sampling/sampling_logp_difference/max": 0.6394569873809814,
|
|
"sampling/importance_sampling_ratio/min": 0.4160086214542389,
|
|
"sampling/importance_sampling_ratio/mean": 0.6889954805374146,
|
|
"sampling/importance_sampling_ratio/max": 1.0844670534133911,
|
|
"kl": 0.0005969375413314992,
|
|
"entropy": 0.03322613658383489,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.383844546973705,
|
|
"epoch": 0.006588541666666667,
|
|
"step": 253
|
|
},
|
|
{
|
|
"loss": 0.37060868740081787,
|
|
"grad_norm": 1.244420051574707,
|
|
"learning_rate": 8.103448275862068e-08,
|
|
"num_tokens": 2712507.0,
|
|
"completions/mean_length": 682.0,
|
|
"completions/min_length": 215.0,
|
|
"completions/max_length": 955.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 530.2000122070312,
|
|
"completions/min_terminated_length": 215.0,
|
|
"completions/max_terminated_length": 931.0,
|
|
"tools/call_frequency": 15.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2984375059604645,
|
|
"rewards/reward_func/std": 0.3637869954109192,
|
|
"reward": 0.2984375059604645,
|
|
"reward_std": 0.3637869656085968,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015168626559898257,
|
|
"sampling/sampling_logp_difference/max": 0.49792909622192383,
|
|
"sampling/importance_sampling_ratio/min": 0.4718916118144989,
|
|
"sampling/importance_sampling_ratio/mean": 0.9109319448471069,
|
|
"sampling/importance_sampling_ratio/max": 1.4801034927368164,
|
|
"kl": 0.0011881237737725314,
|
|
"entropy": 0.035278708615805954,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.514473097398877,
|
|
"epoch": 0.006614583333333333,
|
|
"step": 254
|
|
},
|
|
{
|
|
"loss": -0.09850195050239563,
|
|
"grad_norm": 2.0474836826324463,
|
|
"learning_rate": 7.931034482758621e-08,
|
|
"num_tokens": 2722720.0,
|
|
"completions/mean_length": 591.25,
|
|
"completions/min_length": 160.0,
|
|
"completions/max_length": 916.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 491.5,
|
|
"completions/min_terminated_length": 160.0,
|
|
"completions/max_terminated_length": 916.0,
|
|
"tools/call_frequency": 13.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3439236283302307,
|
|
"rewards/reward_func/std": 0.3603226840496063,
|
|
"reward": 0.3439236283302307,
|
|
"reward_std": 0.36032265424728394,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0031575660686939955,
|
|
"sampling/sampling_logp_difference/max": 0.6529064774513245,
|
|
"sampling/importance_sampling_ratio/min": 0.6643555760383606,
|
|
"sampling/importance_sampling_ratio/mean": 1.0001037120819092,
|
|
"sampling/importance_sampling_ratio/max": 1.410345196723938,
|
|
"kl": 0.00148953553980391,
|
|
"entropy": 0.0610882235923782,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.655700424686074,
|
|
"epoch": 0.006640625,
|
|
"step": 255
|
|
},
|
|
{
|
|
"loss": 0.013258039951324463,
|
|
"grad_norm": 3.41739559173584,
|
|
"learning_rate": 7.758620689655172e-08,
|
|
"num_tokens": 2732190.0,
|
|
"completions/mean_length": 498.875,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 907.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 443.857177734375,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 907.0,
|
|
"tools/call_frequency": 12.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.38072913885116577,
|
|
"rewards/reward_func/std": 0.3595972955226898,
|
|
"reward": 0.38072913885116577,
|
|
"reward_std": 0.3595972955226898,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003151549259200692,
|
|
"sampling/sampling_logp_difference/max": 0.7246572971343994,
|
|
"sampling/importance_sampling_ratio/min": 0.34920036792755127,
|
|
"sampling/importance_sampling_ratio/mean": 0.8995721340179443,
|
|
"sampling/importance_sampling_ratio/max": 1.4638370275497437,
|
|
"kl": 0.0018099043736583553,
|
|
"entropy": 0.06156311463564634,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.497110027819872,
|
|
"epoch": 0.006666666666666667,
|
|
"step": 256
|
|
},
|
|
{
|
|
"loss": -0.0797482579946518,
|
|
"grad_norm": 4.507946968078613,
|
|
"learning_rate": 7.586206896551724e-08,
|
|
"num_tokens": 2741062.0,
|
|
"completions/mean_length": 424.0,
|
|
"completions/min_length": 213.0,
|
|
"completions/max_length": 940.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 350.2857360839844,
|
|
"completions/min_terminated_length": 213.0,
|
|
"completions/max_terminated_length": 883.0,
|
|
"tools/call_frequency": 10.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2614583373069763,
|
|
"rewards/reward_func/std": 0.3155112862586975,
|
|
"reward": 0.2614583373069763,
|
|
"reward_std": 0.3155112862586975,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.004934448283165693,
|
|
"sampling/sampling_logp_difference/max": 1.0398993492126465,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8272535800933838,
|
|
"sampling/importance_sampling_ratio/max": 2.7368805408477783,
|
|
"kl": 0.0013689078032257385,
|
|
"entropy": 0.07731603021966293,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.89331136830151,
|
|
"epoch": 0.0066927083333333335,
|
|
"step": 257
|
|
},
|
|
{
|
|
"loss": -0.20761311054229736,
|
|
"grad_norm": 2.153195858001709,
|
|
"learning_rate": 7.413793103448275e-08,
|
|
"num_tokens": 2750374.0,
|
|
"completions/mean_length": 479.0,
|
|
"completions/min_length": 150.0,
|
|
"completions/max_length": 897.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 343.16668701171875,
|
|
"completions/min_terminated_length": 150.0,
|
|
"completions/max_terminated_length": 875.0,
|
|
"tools/call_frequency": 11.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2526041865348816,
|
|
"rewards/reward_func/std": 0.3410777449607849,
|
|
"reward": 0.2526041865348816,
|
|
"reward_std": 0.3410777449607849,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.004027082119137049,
|
|
"sampling/sampling_logp_difference/max": 0.6608157157897949,
|
|
"sampling/importance_sampling_ratio/min": 0.22011998295783997,
|
|
"sampling/importance_sampling_ratio/mean": 0.6373255848884583,
|
|
"sampling/importance_sampling_ratio/max": 1.1625832319259644,
|
|
"kl": 0.0017636380698604626,
|
|
"entropy": 0.06431283376878127,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.056300962343812,
|
|
"epoch": 0.00671875,
|
|
"step": 258
|
|
},
|
|
{
|
|
"loss": -0.6557965874671936,
|
|
"grad_norm": 2.023653745651245,
|
|
"learning_rate": 7.241379310344827e-08,
|
|
"num_tokens": 2761837.0,
|
|
"completions/mean_length": 746.0,
|
|
"completions/min_length": 178.0,
|
|
"completions/max_length": 957.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 567.0,
|
|
"completions/min_terminated_length": 178.0,
|
|
"completions/max_terminated_length": 941.0,
|
|
"tools/call_frequency": 17.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.012500000186264515,
|
|
"rewards/reward_func/std": 0.06943651288747787,
|
|
"reward": 0.012500000186264515,
|
|
"reward_std": 0.06943650543689728,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0016998458886519074,
|
|
"sampling/sampling_logp_difference/max": 0.564541220664978,
|
|
"sampling/importance_sampling_ratio/min": 0.47505998611450195,
|
|
"sampling/importance_sampling_ratio/mean": 1.1940562725067139,
|
|
"sampling/importance_sampling_ratio/max": 2.6992998123168945,
|
|
"kl": 0.0003063586364078219,
|
|
"entropy": 0.02666190918534994,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.890905115753412,
|
|
"epoch": 0.006744791666666666,
|
|
"step": 259
|
|
},
|
|
{
|
|
"loss": 0.44173330068588257,
|
|
"grad_norm": 2.036552667617798,
|
|
"learning_rate": 7.06896551724138e-08,
|
|
"num_tokens": 2772784.0,
|
|
"completions/mean_length": 683.375,
|
|
"completions/min_length": 242.0,
|
|
"completions/max_length": 985.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 422.5,
|
|
"completions/min_terminated_length": 242.0,
|
|
"completions/max_terminated_length": 897.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3149305582046509,
|
|
"rewards/reward_func/std": 0.3697887063026428,
|
|
"reward": 0.3149305582046509,
|
|
"reward_std": 0.3697887063026428,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020430542062968016,
|
|
"sampling/sampling_logp_difference/max": 0.5628607273101807,
|
|
"sampling/importance_sampling_ratio/min": 0.4036993384361267,
|
|
"sampling/importance_sampling_ratio/mean": 0.9511505961418152,
|
|
"sampling/importance_sampling_ratio/max": 1.322643756866455,
|
|
"kl": 0.0015347807399166413,
|
|
"entropy": 0.04891765970387496,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.700273502618074,
|
|
"epoch": 0.0067708333333333336,
|
|
"step": 260
|
|
},
|
|
{
|
|
"loss": 0.1264086365699768,
|
|
"grad_norm": 2.8752286434173584,
|
|
"learning_rate": 6.89655172413793e-08,
|
|
"num_tokens": 2783519.0,
|
|
"completions/mean_length": 657.625,
|
|
"completions/min_length": 184.0,
|
|
"completions/max_length": 961.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 390.0,
|
|
"completions/min_terminated_length": 184.0,
|
|
"completions/max_terminated_length": 944.0,
|
|
"tools/call_frequency": 15.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.10104166716337204,
|
|
"rewards/reward_func/std": 0.2740944027900696,
|
|
"reward": 0.10104166716337204,
|
|
"reward_std": 0.2740943729877472,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014091178309172392,
|
|
"sampling/sampling_logp_difference/max": 0.43998467922210693,
|
|
"sampling/importance_sampling_ratio/min": 0.42763006687164307,
|
|
"sampling/importance_sampling_ratio/mean": 0.9444577693939209,
|
|
"sampling/importance_sampling_ratio/max": 1.4869866371154785,
|
|
"kl": 0.0012513732897332375,
|
|
"entropy": 0.025096469093114138,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.332572266459465,
|
|
"epoch": 0.006796875,
|
|
"step": 261
|
|
},
|
|
{
|
|
"loss": -0.277569979429245,
|
|
"grad_norm": 2.3277394771575928,
|
|
"learning_rate": 6.724137931034482e-08,
|
|
"num_tokens": 2794200.0,
|
|
"completions/mean_length": 650.75,
|
|
"completions/min_length": 212.0,
|
|
"completions/max_length": 939.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 561.1666870117188,
|
|
"completions/min_terminated_length": 212.0,
|
|
"completions/max_terminated_length": 914.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.13576388359069824,
|
|
"rewards/reward_func/std": 0.21221224963665009,
|
|
"reward": 0.13576388359069824,
|
|
"reward_std": 0.2122122347354889,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018172230338677764,
|
|
"sampling/sampling_logp_difference/max": 0.3898966312408447,
|
|
"sampling/importance_sampling_ratio/min": 0.47671738266944885,
|
|
"sampling/importance_sampling_ratio/mean": 0.8459494709968567,
|
|
"sampling/importance_sampling_ratio/max": 1.9628735780715942,
|
|
"kl": 0.0006720265682815807,
|
|
"entropy": 0.03619183594128117,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.837763560935855,
|
|
"epoch": 0.006822916666666666,
|
|
"step": 262
|
|
},
|
|
{
|
|
"loss": -0.18511050939559937,
|
|
"grad_norm": 1.7982022762298584,
|
|
"learning_rate": 6.551724137931034e-08,
|
|
"num_tokens": 2804655.0,
|
|
"completions/mean_length": 622.125,
|
|
"completions/min_length": 102.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 447.8000183105469,
|
|
"completions/min_terminated_length": 102.0,
|
|
"completions/max_terminated_length": 923.0,
|
|
"tools/call_frequency": 15.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.06458333879709244,
|
|
"rewards/reward_func/std": 0.23510171473026276,
|
|
"reward": 0.06458333879709244,
|
|
"reward_std": 0.23510172963142395,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001918208203278482,
|
|
"sampling/sampling_logp_difference/max": 0.455249547958374,
|
|
"sampling/importance_sampling_ratio/min": 0.30626973509788513,
|
|
"sampling/importance_sampling_ratio/mean": 0.8503973484039307,
|
|
"sampling/importance_sampling_ratio/max": 1.4349063634872437,
|
|
"kl": 0.0011852581228595227,
|
|
"entropy": 0.035916958819143474,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.336849477142096,
|
|
"epoch": 0.006848958333333334,
|
|
"step": 263
|
|
},
|
|
{
|
|
"loss": -0.14613056182861328,
|
|
"grad_norm": 1.8665395975112915,
|
|
"learning_rate": 6.379310344827587e-08,
|
|
"num_tokens": 2815483.0,
|
|
"completions/mean_length": 667.75,
|
|
"completions/min_length": 149.0,
|
|
"completions/max_length": 952.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 414.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"completions/max_terminated_length": 952.0,
|
|
"tools/call_frequency": 15.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.09062499552965164,
|
|
"rewards/reward_func/std": 0.19997209310531616,
|
|
"reward": 0.09062499552965164,
|
|
"reward_std": 0.19997210800647736,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017863503890112042,
|
|
"sampling/sampling_logp_difference/max": 0.5616294145584106,
|
|
"sampling/importance_sampling_ratio/min": 0.7407109141349792,
|
|
"sampling/importance_sampling_ratio/mean": 1.1001992225646973,
|
|
"sampling/importance_sampling_ratio/max": 1.6041345596313477,
|
|
"kl": 0.0004723484832993563,
|
|
"entropy": 0.03195500519359484,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.826285265386105,
|
|
"epoch": 0.006875,
|
|
"step": 264
|
|
},
|
|
{
|
|
"loss": 0.3568809926509857,
|
|
"grad_norm": 4.104587078094482,
|
|
"learning_rate": 6.206896551724137e-08,
|
|
"num_tokens": 2825925.0,
|
|
"completions/mean_length": 619.625,
|
|
"completions/min_length": 233.0,
|
|
"completions/max_length": 999.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 565.4285888671875,
|
|
"completions/min_terminated_length": 233.0,
|
|
"completions/max_terminated_length": 944.0,
|
|
"tools/call_frequency": 14.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21822915971279144,
|
|
"rewards/reward_func/std": 0.3184936046600342,
|
|
"reward": 0.21822915971279144,
|
|
"reward_std": 0.31849363446235657,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0027973388787359,
|
|
"sampling/sampling_logp_difference/max": 0.5645407438278198,
|
|
"sampling/importance_sampling_ratio/min": 0.4651997685432434,
|
|
"sampling/importance_sampling_ratio/mean": 0.988208532333374,
|
|
"sampling/importance_sampling_ratio/max": 2.5567588806152344,
|
|
"kl": 0.0009338553704765218,
|
|
"entropy": 0.057810735306702554,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.37744397111237,
|
|
"epoch": 0.0069010416666666664,
|
|
"step": 265
|
|
},
|
|
{
|
|
"loss": 0.40765056014060974,
|
|
"grad_norm": 1.3078607320785522,
|
|
"learning_rate": 6.03448275862069e-08,
|
|
"num_tokens": 2837502.0,
|
|
"completions/mean_length": 762.0,
|
|
"completions/min_length": 272.0,
|
|
"completions/max_length": 946.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 673.6000366210938,
|
|
"completions/min_terminated_length": 272.0,
|
|
"completions/max_terminated_length": 946.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17425596714019775,
|
|
"rewards/reward_func/std": 0.23144622147083282,
|
|
"reward": 0.17425596714019775,
|
|
"reward_std": 0.23144620656967163,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018892992520704865,
|
|
"sampling/sampling_logp_difference/max": 0.4239692687988281,
|
|
"sampling/importance_sampling_ratio/min": 0.328146368265152,
|
|
"sampling/importance_sampling_ratio/mean": 0.9215643405914307,
|
|
"sampling/importance_sampling_ratio/max": 1.4068865776062012,
|
|
"kl": 0.000668848999112015,
|
|
"entropy": 0.03496758942492306,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.098803805187345,
|
|
"epoch": 0.006927083333333334,
|
|
"step": 266
|
|
},
|
|
{
|
|
"loss": 0.1896815001964569,
|
|
"grad_norm": 4.5845794677734375,
|
|
"learning_rate": 5.862068965517241e-08,
|
|
"num_tokens": 2848296.0,
|
|
"completions/mean_length": 665.125,
|
|
"completions/min_length": 228.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 504.6000061035156,
|
|
"completions/min_terminated_length": 228.0,
|
|
"completions/max_terminated_length": 920.0,
|
|
"tools/call_frequency": 15.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18437500298023224,
|
|
"rewards/reward_func/std": 0.19408094882965088,
|
|
"reward": 0.18437500298023224,
|
|
"reward_std": 0.19408094882965088,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002171938307583332,
|
|
"sampling/sampling_logp_difference/max": 0.3082820177078247,
|
|
"sampling/importance_sampling_ratio/min": 0.4200004041194916,
|
|
"sampling/importance_sampling_ratio/mean": 0.9132033586502075,
|
|
"sampling/importance_sampling_ratio/max": 1.9988583326339722,
|
|
"kl": 0.0008161162290889479,
|
|
"entropy": 0.036173399072140455,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.643076850101352,
|
|
"epoch": 0.006953125,
|
|
"step": 267
|
|
},
|
|
{
|
|
"loss": 0.3302306532859802,
|
|
"grad_norm": 4.22349739074707,
|
|
"learning_rate": 5.689655172413793e-08,
|
|
"num_tokens": 2858501.0,
|
|
"completions/mean_length": 590.5,
|
|
"completions/min_length": 174.0,
|
|
"completions/max_length": 970.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/mean_terminated_length": 590.5,
|
|
"completions/min_terminated_length": 174.0,
|
|
"completions/max_terminated_length": 970.0,
|
|
"tools/call_frequency": 14.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21562500298023224,
|
|
"rewards/reward_func/std": 0.34665173292160034,
|
|
"reward": 0.21562500298023224,
|
|
"reward_std": 0.34665170311927795,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0023851958103477955,
|
|
"sampling/sampling_logp_difference/max": 0.6608167886734009,
|
|
"sampling/importance_sampling_ratio/min": 0.5553617477416992,
|
|
"sampling/importance_sampling_ratio/mean": 1.260180950164795,
|
|
"sampling/importance_sampling_ratio/max": 2.3708157539367676,
|
|
"kl": 0.001093789593795691,
|
|
"entropy": 0.04040538886329159,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.844992738217115,
|
|
"epoch": 0.0069791666666666665,
|
|
"step": 268
|
|
},
|
|
{
|
|
"loss": -0.38896840810775757,
|
|
"grad_norm": 1.0138412714004517,
|
|
"learning_rate": 5.517241379310345e-08,
|
|
"num_tokens": 2869746.0,
|
|
"completions/mean_length": 721.25,
|
|
"completions/min_length": 176.0,
|
|
"completions/max_length": 915.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 539.75,
|
|
"completions/min_terminated_length": 176.0,
|
|
"completions/max_terminated_length": 896.0,
|
|
"tools/call_frequency": 17.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.012500000186264515,
|
|
"rewards/reward_func/std": 0.06943651288747787,
|
|
"reward": 0.012500000186264515,
|
|
"reward_std": 0.06943650543689728,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001457652193494141,
|
|
"sampling/sampling_logp_difference/max": 0.4540271759033203,
|
|
"sampling/importance_sampling_ratio/min": 0.35124242305755615,
|
|
"sampling/importance_sampling_ratio/mean": 0.743012011051178,
|
|
"sampling/importance_sampling_ratio/max": 1.1275038719177246,
|
|
"kl": 0.0002600864579562767,
|
|
"entropy": 0.028812801523599774,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.25513774342835,
|
|
"epoch": 0.007005208333333333,
|
|
"step": 269
|
|
},
|
|
{
|
|
"loss": 0.2891576588153839,
|
|
"grad_norm": 2.992758274078369,
|
|
"learning_rate": 5.344827586206897e-08,
|
|
"num_tokens": 2881377.0,
|
|
"completions/mean_length": 768.375,
|
|
"completions/min_length": 239.0,
|
|
"completions/max_length": 961.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 674.6000366210938,
|
|
"completions/min_terminated_length": 239.0,
|
|
"completions/max_terminated_length": 961.0,
|
|
"tools/call_frequency": 17.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1875,
|
|
"rewards/reward_func/std": 0.20712746679782867,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.20712745189666748,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0020916855428367853,
|
|
"sampling/sampling_logp_difference/max": 0.7046945095062256,
|
|
"sampling/importance_sampling_ratio/min": 0.3687598407268524,
|
|
"sampling/importance_sampling_ratio/mean": 0.9652462005615234,
|
|
"sampling/importance_sampling_ratio/max": 1.4637874364852905,
|
|
"kl": 0.0010742664144345326,
|
|
"entropy": 0.04226671124342829,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.450607622042298,
|
|
"epoch": 0.00703125,
|
|
"step": 270
|
|
},
|
|
{
|
|
"loss": 0.27633583545684814,
|
|
"grad_norm": 3.2457218170166016,
|
|
"learning_rate": 5.172413793103448e-08,
|
|
"num_tokens": 2893309.0,
|
|
"completions/mean_length": 807.0,
|
|
"completions/min_length": 236.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 727.75,
|
|
"completions/min_terminated_length": 236.0,
|
|
"completions/max_terminated_length": 920.0,
|
|
"tools/call_frequency": 19.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2671875059604645,
|
|
"rewards/reward_func/std": 0.3069179058074951,
|
|
"reward": 0.2671875059604645,
|
|
"reward_std": 0.3069179058074951,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017180058639496565,
|
|
"sampling/sampling_logp_difference/max": 1.2919940948486328,
|
|
"sampling/importance_sampling_ratio/min": 0.11937947571277618,
|
|
"sampling/importance_sampling_ratio/mean": 0.8416699767112732,
|
|
"sampling/importance_sampling_ratio/max": 1.6045291423797607,
|
|
"kl": 0.0010486689816389116,
|
|
"entropy": 0.020860985037870705,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.980812074616551,
|
|
"epoch": 0.007057291666666667,
|
|
"step": 271
|
|
},
|
|
{
|
|
"loss": 0.27636438608169556,
|
|
"grad_norm": 3.325765371322632,
|
|
"learning_rate": 5e-08,
|
|
"num_tokens": 2902837.0,
|
|
"completions/mean_length": 505.0,
|
|
"completions/min_length": 203.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 366.8333435058594,
|
|
"completions/min_terminated_length": 203.0,
|
|
"completions/max_terminated_length": 871.0,
|
|
"tools/call_frequency": 12.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23281250894069672,
|
|
"rewards/reward_func/std": 0.38271453976631165,
|
|
"reward": 0.23281250894069672,
|
|
"reward_std": 0.38271453976631165,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003951175604015589,
|
|
"sampling/sampling_logp_difference/max": 0.5378977656364441,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.9705746173858643,
|
|
"sampling/importance_sampling_ratio/max": 2.3261709213256836,
|
|
"kl": 0.0017916684828378493,
|
|
"entropy": 0.055625192588195205,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.17064710892737,
|
|
"epoch": 0.007083333333333333,
|
|
"step": 272
|
|
},
|
|
{
|
|
"loss": -0.25748342275619507,
|
|
"grad_norm": 2.7412872314453125,
|
|
"learning_rate": 4.827586206896551e-08,
|
|
"num_tokens": 2912894.0,
|
|
"completions/mean_length": 571.0,
|
|
"completions/min_length": 173.0,
|
|
"completions/max_length": 920.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 458.5,
|
|
"completions/min_terminated_length": 173.0,
|
|
"completions/max_terminated_length": 916.0,
|
|
"tools/call_frequency": 14.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.17343750596046448,
|
|
"rewards/reward_func/std": 0.36058223247528076,
|
|
"reward": 0.17343750596046448,
|
|
"reward_std": 0.36058223247528076,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0027384269051253796,
|
|
"sampling/sampling_logp_difference/max": 0.6119542121887207,
|
|
"sampling/importance_sampling_ratio/min": 0.28937414288520813,
|
|
"sampling/importance_sampling_ratio/mean": 1.0763660669326782,
|
|
"sampling/importance_sampling_ratio/max": 2.402567148208618,
|
|
"kl": 0.0008920743421185762,
|
|
"entropy": 0.04990255582379177,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.799800297245383,
|
|
"epoch": 0.007109375,
|
|
"step": 273
|
|
},
|
|
{
|
|
"loss": 0.19490329921245575,
|
|
"grad_norm": 5.123165607452393,
|
|
"learning_rate": 4.6551724137931034e-08,
|
|
"num_tokens": 2921564.0,
|
|
"completions/mean_length": 399.125,
|
|
"completions/min_length": 142.0,
|
|
"completions/max_length": 911.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 229.83334350585938,
|
|
"completions/min_terminated_length": 142.0,
|
|
"completions/max_terminated_length": 296.0,
|
|
"tools/call_frequency": 9.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.371527761220932,
|
|
"rewards/reward_func/std": 0.43715938925743103,
|
|
"reward": 0.371527761220932,
|
|
"reward_std": 0.4371594190597534,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003352103056386113,
|
|
"sampling/sampling_logp_difference/max": 0.6249872446060181,
|
|
"sampling/importance_sampling_ratio/min": 0.2631630599498749,
|
|
"sampling/importance_sampling_ratio/mean": 0.904085636138916,
|
|
"sampling/importance_sampling_ratio/max": 1.3664675951004028,
|
|
"kl": 0.001760008211931563,
|
|
"entropy": 0.06161305378191173,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.932975325733423,
|
|
"epoch": 0.007135416666666667,
|
|
"step": 274
|
|
},
|
|
{
|
|
"loss": -0.09521540254354477,
|
|
"grad_norm": 1.3611477613449097,
|
|
"learning_rate": 4.482758620689655e-08,
|
|
"num_tokens": 2932952.0,
|
|
"completions/mean_length": 739.125,
|
|
"completions/min_length": 253.0,
|
|
"completions/max_length": 937.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 455.66668701171875,
|
|
"completions/min_terminated_length": 253.0,
|
|
"completions/max_terminated_length": 851.0,
|
|
"tools/call_frequency": 17.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.28697916865348816,
|
|
"rewards/reward_func/std": 0.34756407141685486,
|
|
"reward": 0.28697916865348816,
|
|
"reward_std": 0.34756407141685486,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0018288629362359643,
|
|
"sampling/sampling_logp_difference/max": 0.48116183280944824,
|
|
"sampling/importance_sampling_ratio/min": 0.2948235273361206,
|
|
"sampling/importance_sampling_ratio/mean": 0.8070800304412842,
|
|
"sampling/importance_sampling_ratio/max": 1.3126376867294312,
|
|
"kl": 0.0006546253680426162,
|
|
"entropy": 0.029251295782160014,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.708067102357745,
|
|
"epoch": 0.007161458333333333,
|
|
"step": 275
|
|
},
|
|
{
|
|
"loss": -0.052616994827985764,
|
|
"grad_norm": 1.37532639503479,
|
|
"learning_rate": 4.310344827586207e-08,
|
|
"num_tokens": 2943793.0,
|
|
"completions/mean_length": 670.75,
|
|
"completions/min_length": 227.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 422.25,
|
|
"completions/min_terminated_length": 227.0,
|
|
"completions/max_terminated_length": 913.0,
|
|
"tools/call_frequency": 16.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.22336310148239136,
|
|
"rewards/reward_func/std": 0.33114317059516907,
|
|
"reward": 0.22336310148239136,
|
|
"reward_std": 0.33114317059516907,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013926505343988538,
|
|
"sampling/sampling_logp_difference/max": 0.5541767477989197,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.7382208704948425,
|
|
"sampling/importance_sampling_ratio/max": 1.2540351152420044,
|
|
"kl": 0.0009677112518886588,
|
|
"entropy": 0.03122883354080841,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.614202577620745,
|
|
"epoch": 0.0071875,
|
|
"step": 276
|
|
},
|
|
{
|
|
"loss": 0.44836291670799255,
|
|
"grad_norm": 2.7302050590515137,
|
|
"learning_rate": 4.1379310344827585e-08,
|
|
"num_tokens": 2953213.0,
|
|
"completions/mean_length": 492.625,
|
|
"completions/min_length": 208.0,
|
|
"completions/max_length": 933.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 429.71429443359375,
|
|
"completions/min_terminated_length": 208.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 12.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23106059432029724,
|
|
"rewards/reward_func/std": 0.3493136763572693,
|
|
"reward": 0.23106059432029724,
|
|
"reward_std": 0.3493136465549469,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0022099760826677084,
|
|
"sampling/sampling_logp_difference/max": 1.0575494766235352,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 0.8936549425125122,
|
|
"sampling/importance_sampling_ratio/max": 1.4195703268051147,
|
|
"kl": 0.0011621098611840353,
|
|
"entropy": 0.04978325011325069,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.627884790301323,
|
|
"epoch": 0.007213541666666667,
|
|
"step": 277
|
|
},
|
|
{
|
|
"loss": 0.24297398328781128,
|
|
"grad_norm": 2.2487525939941406,
|
|
"learning_rate": 3.9655172413793106e-08,
|
|
"num_tokens": 2963446.0,
|
|
"completions/mean_length": 593.875,
|
|
"completions/min_length": 179.0,
|
|
"completions/max_length": 941.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 485.0,
|
|
"completions/min_terminated_length": 179.0,
|
|
"completions/max_terminated_length": 941.0,
|
|
"tools/call_frequency": 13.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23482143878936768,
|
|
"rewards/reward_func/std": 0.39250797033309937,
|
|
"reward": 0.23482143878936768,
|
|
"reward_std": 0.392507940530777,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.00318559305742383,
|
|
"sampling/sampling_logp_difference/max": 0.7341852188110352,
|
|
"sampling/importance_sampling_ratio/min": 0.3446553945541382,
|
|
"sampling/importance_sampling_ratio/mean": 0.8051729202270508,
|
|
"sampling/importance_sampling_ratio/max": 1.183397650718689,
|
|
"kl": 0.0019130404743918916,
|
|
"entropy": 0.05361370765604079,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.946444088593125,
|
|
"epoch": 0.007239583333333333,
|
|
"step": 278
|
|
},
|
|
{
|
|
"loss": 0.049527011811733246,
|
|
"grad_norm": 1.0780048370361328,
|
|
"learning_rate": 3.793103448275862e-08,
|
|
"num_tokens": 2974770.0,
|
|
"completions/mean_length": 729.875,
|
|
"completions/min_length": 165.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 665.1666870117188,
|
|
"completions/min_terminated_length": 165.0,
|
|
"completions/max_terminated_length": 923.0,
|
|
"tools/call_frequency": 17.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1093750074505806,
|
|
"rewards/reward_func/std": 0.23449701070785522,
|
|
"reward": 0.1093750074505806,
|
|
"reward_std": 0.23449699580669403,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013216757215559483,
|
|
"sampling/sampling_logp_difference/max": 0.7311468720436096,
|
|
"sampling/importance_sampling_ratio/min": 0.4700555205345154,
|
|
"sampling/importance_sampling_ratio/mean": 1.12735116481781,
|
|
"sampling/importance_sampling_ratio/max": 2.2702040672302246,
|
|
"kl": 0.0006824223173680366,
|
|
"entropy": 0.01997874944936484,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.056817376986146,
|
|
"epoch": 0.007265625,
|
|
"step": 279
|
|
},
|
|
{
|
|
"loss": 0.19637449085712433,
|
|
"grad_norm": 3.441924571990967,
|
|
"learning_rate": 3.6206896551724136e-08,
|
|
"num_tokens": 2985415.0,
|
|
"completions/mean_length": 646.625,
|
|
"completions/min_length": 170.0,
|
|
"completions/max_length": 940.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 386.5,
|
|
"completions/min_terminated_length": 170.0,
|
|
"completions/max_terminated_length": 900.0,
|
|
"tools/call_frequency": 16.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.21979166567325592,
|
|
"rewards/reward_func/std": 0.2963715195655823,
|
|
"reward": 0.21979166567325592,
|
|
"reward_std": 0.2963714897632599,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013740264112129807,
|
|
"sampling/sampling_logp_difference/max": 0.6405763626098633,
|
|
"sampling/importance_sampling_ratio/min": 0.6045573353767395,
|
|
"sampling/importance_sampling_ratio/mean": 1.150895595550537,
|
|
"sampling/importance_sampling_ratio/max": 2.006645679473877,
|
|
"kl": 0.001668890991822991,
|
|
"entropy": 0.0263326974818483,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.505161127075553,
|
|
"epoch": 0.007291666666666667,
|
|
"step": 280
|
|
},
|
|
{
|
|
"loss": 0.018024805933237076,
|
|
"grad_norm": 1.5469861030578613,
|
|
"learning_rate": 3.448275862068965e-08,
|
|
"num_tokens": 2996864.0,
|
|
"completions/mean_length": 745.75,
|
|
"completions/min_length": 192.0,
|
|
"completions/max_length": 978.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 625.0,
|
|
"completions/min_terminated_length": 192.0,
|
|
"completions/max_terminated_length": 923.0,
|
|
"tools/call_frequency": 16.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2863839268684387,
|
|
"rewards/reward_func/std": 0.38876572251319885,
|
|
"reward": 0.2863839268684387,
|
|
"reward_std": 0.38876572251319885,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013781175948679447,
|
|
"sampling/sampling_logp_difference/max": 0.42461490631103516,
|
|
"sampling/importance_sampling_ratio/min": 0.5483893156051636,
|
|
"sampling/importance_sampling_ratio/mean": 1.0245462656021118,
|
|
"sampling/importance_sampling_ratio/max": 1.4001654386520386,
|
|
"kl": 0.0009248566047972417,
|
|
"entropy": 0.03198443935252726,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.037216685712337,
|
|
"epoch": 0.007317708333333333,
|
|
"step": 281
|
|
},
|
|
{
|
|
"loss": 0.5122004151344299,
|
|
"grad_norm": 3.775883436203003,
|
|
"learning_rate": 3.275862068965517e-08,
|
|
"num_tokens": 3007641.0,
|
|
"completions/mean_length": 661.75,
|
|
"completions/min_length": 204.0,
|
|
"completions/max_length": 935.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 507.8000183105469,
|
|
"completions/min_terminated_length": 204.0,
|
|
"completions/max_terminated_length": 915.0,
|
|
"tools/call_frequency": 15.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.35712260007858276,
|
|
"rewards/reward_func/std": 0.3555957078933716,
|
|
"reward": 0.35712260007858276,
|
|
"reward_std": 0.3555956780910492,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001509360154159367,
|
|
"sampling/sampling_logp_difference/max": 0.3775310516357422,
|
|
"sampling/importance_sampling_ratio/min": 0.4879939556121826,
|
|
"sampling/importance_sampling_ratio/mean": 1.0301026105880737,
|
|
"sampling/importance_sampling_ratio/max": 1.6235891580581665,
|
|
"kl": 0.0020899172877761885,
|
|
"entropy": 0.034324785810895264,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.507269453257322,
|
|
"epoch": 0.00734375,
|
|
"step": 282
|
|
},
|
|
{
|
|
"loss": -0.007321242243051529,
|
|
"grad_norm": 2.200709104537964,
|
|
"learning_rate": 3.103448275862069e-08,
|
|
"num_tokens": 3019104.0,
|
|
"completions/mean_length": 747.875,
|
|
"completions/min_length": 184.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 253.5,
|
|
"completions/min_terminated_length": 184.0,
|
|
"completions/max_terminated_length": 323.0,
|
|
"tools/call_frequency": 17.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.12656250596046448,
|
|
"rewards/reward_func/std": 0.1950775533914566,
|
|
"reward": 0.12656250596046448,
|
|
"reward_std": 0.1950775384902954,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019098685588687658,
|
|
"sampling/sampling_logp_difference/max": 0.527026891708374,
|
|
"sampling/importance_sampling_ratio/min": 0.6225882172584534,
|
|
"sampling/importance_sampling_ratio/mean": 0.9106945991516113,
|
|
"sampling/importance_sampling_ratio/max": 1.2817972898483276,
|
|
"kl": 0.000464978832496854,
|
|
"entropy": 0.02900763595243916,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.26254614815116,
|
|
"epoch": 0.007369791666666667,
|
|
"step": 283
|
|
},
|
|
{
|
|
"loss": -0.2766661047935486,
|
|
"grad_norm": 0.8259903192520142,
|
|
"learning_rate": 2.9310344827586205e-08,
|
|
"num_tokens": 3031198.0,
|
|
"completions/mean_length": 825.25,
|
|
"completions/min_length": 143.0,
|
|
"completions/max_length": 993.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 678.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"completions/max_terminated_length": 993.0,
|
|
"tools/call_frequency": 19.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0014634705148637295,
|
|
"sampling/sampling_logp_difference/max": 0.7733474969863892,
|
|
"sampling/importance_sampling_ratio/min": 0.42173951864242554,
|
|
"sampling/importance_sampling_ratio/mean": 0.8448672294616699,
|
|
"sampling/importance_sampling_ratio/max": 1.6815975904464722,
|
|
"kl": 0.0004461145817913348,
|
|
"entropy": 0.014416073216125369,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.575233304873109,
|
|
"epoch": 0.007395833333333333,
|
|
"step": 284
|
|
},
|
|
{
|
|
"loss": 0.008039243519306183,
|
|
"grad_norm": 3.0976474285125732,
|
|
"learning_rate": 2.7586206896551723e-08,
|
|
"num_tokens": 3040754.0,
|
|
"completions/mean_length": 509.25,
|
|
"completions/min_length": 203.0,
|
|
"completions/max_length": 944.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/mean_terminated_length": 453.4285888671875,
|
|
"completions/min_terminated_length": 203.0,
|
|
"completions/max_terminated_length": 944.0,
|
|
"tools/call_frequency": 12.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3973214626312256,
|
|
"rewards/reward_func/std": 0.3356952965259552,
|
|
"reward": 0.3973214626312256,
|
|
"reward_std": 0.3356952965259552,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.003147474257275462,
|
|
"sampling/sampling_logp_difference/max": 0.6191270351409912,
|
|
"sampling/importance_sampling_ratio/min": 0.3229880928993225,
|
|
"sampling/importance_sampling_ratio/mean": 1.1407883167266846,
|
|
"sampling/importance_sampling_ratio/max": 2.435445785522461,
|
|
"kl": 0.0019564594640542055,
|
|
"entropy": 0.062184353882912546,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.254318058490753,
|
|
"epoch": 0.007421875,
|
|
"step": 285
|
|
},
|
|
{
|
|
"loss": -0.18671730160713196,
|
|
"grad_norm": 5.126163959503174,
|
|
"learning_rate": 2.586206896551724e-08,
|
|
"num_tokens": 3050999.0,
|
|
"completions/mean_length": 594.75,
|
|
"completions/min_length": 236.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 265.0,
|
|
"completions/min_terminated_length": 236.0,
|
|
"completions/max_terminated_length": 326.0,
|
|
"tools/call_frequency": 14.125,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.012500000186264515,
|
|
"rewards/reward_func/std": 0.1060660183429718,
|
|
"reward": 0.012500000186264515,
|
|
"reward_std": 0.1060660183429718,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0021953920368105173,
|
|
"sampling/sampling_logp_difference/max": 1.236915111541748,
|
|
"sampling/importance_sampling_ratio/min": 0.5053666830062866,
|
|
"sampling/importance_sampling_ratio/mean": 1.248411774635315,
|
|
"sampling/importance_sampling_ratio/max": 2.9863932132720947,
|
|
"kl": 0.0010684548024073592,
|
|
"entropy": 0.03655444865580648,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.331141512840986,
|
|
"epoch": 0.007447916666666667,
|
|
"step": 286
|
|
},
|
|
{
|
|
"loss": -0.0025546960532665253,
|
|
"grad_norm": 4.6264328956604,
|
|
"learning_rate": 2.4137931034482756e-08,
|
|
"num_tokens": 3061051.0,
|
|
"completions/mean_length": 571.375,
|
|
"completions/min_length": 183.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 361.0,
|
|
"completions/min_terminated_length": 183.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 13.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.07916666567325592,
|
|
"rewards/reward_func/std": 0.27454873919487,
|
|
"reward": 0.07916666567325592,
|
|
"reward_std": 0.2745487093925476,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002208986319601536,
|
|
"sampling/sampling_logp_difference/max": 1.5466638803482056,
|
|
"sampling/importance_sampling_ratio/min": 0.12674658000469208,
|
|
"sampling/importance_sampling_ratio/mean": 0.7796528935432434,
|
|
"sampling/importance_sampling_ratio/max": 1.1996136903762817,
|
|
"kl": 0.0008230032875644611,
|
|
"entropy": 0.039714706246741116,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.757593987509608,
|
|
"epoch": 0.007473958333333333,
|
|
"step": 287
|
|
},
|
|
{
|
|
"loss": 0.3533742427825928,
|
|
"grad_norm": 4.149740695953369,
|
|
"learning_rate": 2.2413793103448274e-08,
|
|
"num_tokens": 3071022.0,
|
|
"completions/mean_length": 561.875,
|
|
"completions/min_length": 140.0,
|
|
"completions/max_length": 938.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 358.8000183105469,
|
|
"completions/min_terminated_length": 140.0,
|
|
"completions/max_terminated_length": 907.0,
|
|
"tools/call_frequency": 13.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2718749940395355,
|
|
"rewards/reward_func/std": 0.3097277879714966,
|
|
"reward": 0.2718749940395355,
|
|
"reward_std": 0.3097277581691742,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0023425635881721973,
|
|
"sampling/sampling_logp_difference/max": 0.44146013259887695,
|
|
"sampling/importance_sampling_ratio/min": 0.34544721245765686,
|
|
"sampling/importance_sampling_ratio/mean": 0.9381816387176514,
|
|
"sampling/importance_sampling_ratio/max": 1.7718031406402588,
|
|
"kl": 0.0021283248959207413,
|
|
"entropy": 0.046396831050515175,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.510498465970159,
|
|
"epoch": 0.0075,
|
|
"step": 288
|
|
},
|
|
{
|
|
"loss": 0.2104550004005432,
|
|
"grad_norm": 7.859919548034668,
|
|
"learning_rate": 2.0689655172413793e-08,
|
|
"num_tokens": 3081244.0,
|
|
"completions/mean_length": 594.0,
|
|
"completions/min_length": 231.0,
|
|
"completions/max_length": 934.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/mean_terminated_length": 594.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"completions/max_terminated_length": 934.0,
|
|
"tools/call_frequency": 14.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2770833373069763,
|
|
"rewards/reward_func/std": 0.36706554889678955,
|
|
"reward": 0.2770833373069763,
|
|
"reward_std": 0.36706554889678955,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002491989638656378,
|
|
"sampling/sampling_logp_difference/max": 0.49311524629592896,
|
|
"sampling/importance_sampling_ratio/min": 0.7419575452804565,
|
|
"sampling/importance_sampling_ratio/mean": 1.1921906471252441,
|
|
"sampling/importance_sampling_ratio/max": 1.920769214630127,
|
|
"kl": 0.001272853103273519,
|
|
"entropy": 0.058320170210208744,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.007898945361376,
|
|
"epoch": 0.007526041666666667,
|
|
"step": 289
|
|
},
|
|
{
|
|
"loss": -0.12859219312667847,
|
|
"grad_norm": 4.777729034423828,
|
|
"learning_rate": 1.896551724137931e-08,
|
|
"num_tokens": 3092274.0,
|
|
"completions/mean_length": 694.375,
|
|
"completions/min_length": 171.0,
|
|
"completions/max_length": 981.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 548.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"completions/max_terminated_length": 913.0,
|
|
"tools/call_frequency": 16.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18645834922790527,
|
|
"rewards/reward_func/std": 0.3047877848148346,
|
|
"reward": 0.18645834922790527,
|
|
"reward_std": 0.3047877848148346,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001832861453294754,
|
|
"sampling/sampling_logp_difference/max": 0.4882345199584961,
|
|
"sampling/importance_sampling_ratio/min": 0.448787122964859,
|
|
"sampling/importance_sampling_ratio/mean": 1.178660273551941,
|
|
"sampling/importance_sampling_ratio/max": 2.6907899379730225,
|
|
"kl": 0.0011843626380141359,
|
|
"entropy": 0.026539937942288816,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.800126483663917,
|
|
"epoch": 0.007552083333333333,
|
|
"step": 290
|
|
},
|
|
{
|
|
"loss": 0.16858543455600739,
|
|
"grad_norm": 1.410737156867981,
|
|
"learning_rate": 1.7241379310344825e-08,
|
|
"num_tokens": 3102952.0,
|
|
"completions/mean_length": 650.5,
|
|
"completions/min_length": 168.0,
|
|
"completions/max_length": 938.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 238.6666717529297,
|
|
"completions/min_terminated_length": 168.0,
|
|
"completions/max_terminated_length": 292.0,
|
|
"tools/call_frequency": 15.875,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1510416567325592,
|
|
"rewards/reward_func/std": 0.19181865453720093,
|
|
"reward": 0.1510416567325592,
|
|
"reward_std": 0.19181865453720093,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019944224040955305,
|
|
"sampling/sampling_logp_difference/max": 1.2656278610229492,
|
|
"sampling/importance_sampling_ratio/min": 0.17246422171592712,
|
|
"sampling/importance_sampling_ratio/mean": 0.6297210454940796,
|
|
"sampling/importance_sampling_ratio/max": 1.3021814823150635,
|
|
"kl": 0.000575417556319735,
|
|
"entropy": 0.027456628216896206,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.721895013004541,
|
|
"epoch": 0.007578125,
|
|
"step": 291
|
|
},
|
|
{
|
|
"loss": 0.0921265035867691,
|
|
"grad_norm": 3.025827169418335,
|
|
"learning_rate": 1.5517241379310344e-08,
|
|
"num_tokens": 3113739.0,
|
|
"completions/mean_length": 662.625,
|
|
"completions/min_length": 237.0,
|
|
"completions/max_length": 979.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/mean_terminated_length": 566.6666870117188,
|
|
"completions/min_terminated_length": 237.0,
|
|
"completions/max_terminated_length": 912.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.23854166269302368,
|
|
"rewards/reward_func/std": 0.326596736907959,
|
|
"reward": 0.23854166269302368,
|
|
"reward_std": 0.326596736907959,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0017985495505854487,
|
|
"sampling/sampling_logp_difference/max": 0.7246577739715576,
|
|
"sampling/importance_sampling_ratio/min": 0.40337303280830383,
|
|
"sampling/importance_sampling_ratio/mean": 0.987341046333313,
|
|
"sampling/importance_sampling_ratio/max": 2.152456521987915,
|
|
"kl": 0.0010520498854020843,
|
|
"entropy": 0.034149241633713245,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.617316126823425,
|
|
"epoch": 0.007604166666666667,
|
|
"step": 292
|
|
},
|
|
{
|
|
"loss": 0.05111709237098694,
|
|
"grad_norm": 2.812838077545166,
|
|
"learning_rate": 1.3793103448275862e-08,
|
|
"num_tokens": 3126077.0,
|
|
"completions/mean_length": 856.0,
|
|
"completions/min_length": 316.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/clipped_ratio": 0.75,
|
|
"completions/mean_terminated_length": 602.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"completions/max_terminated_length": 888.0,
|
|
"tools/call_frequency": 18.25,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2022569477558136,
|
|
"rewards/reward_func/std": 0.20715798437595367,
|
|
"reward": 0.2022569477558136,
|
|
"reward_std": 0.20715798437595367,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0019341044826433063,
|
|
"sampling/sampling_logp_difference/max": 1.2133357524871826,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/importance_sampling_ratio/mean": 1.1859285831451416,
|
|
"sampling/importance_sampling_ratio/max": 2.5516839027404785,
|
|
"kl": 0.0004263872569936211,
|
|
"entropy": 0.0257022074656561,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.2743882201612,
|
|
"epoch": 0.0076302083333333335,
|
|
"step": 293
|
|
},
|
|
{
|
|
"loss": 0.07875598222017288,
|
|
"grad_norm": 4.678696155548096,
|
|
"learning_rate": 1.2068965517241378e-08,
|
|
"num_tokens": 3136179.0,
|
|
"completions/mean_length": 576.375,
|
|
"completions/min_length": 184.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 368.0,
|
|
"completions/min_terminated_length": 184.0,
|
|
"completions/max_terminated_length": 917.0,
|
|
"tools/call_frequency": 13.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.2673611044883728,
|
|
"rewards/reward_func/std": 0.3483220636844635,
|
|
"reward": 0.2673611044883728,
|
|
"reward_std": 0.3483220636844635,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0027078755665570498,
|
|
"sampling/sampling_logp_difference/max": 1.341698408126831,
|
|
"sampling/importance_sampling_ratio/min": 0.12779374420642853,
|
|
"sampling/importance_sampling_ratio/mean": 1.024421215057373,
|
|
"sampling/importance_sampling_ratio/max": 1.789402723312378,
|
|
"kl": 0.002127788673078612,
|
|
"entropy": 0.05563342903042212,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.405589642003179,
|
|
"epoch": 0.00765625,
|
|
"step": 294
|
|
},
|
|
{
|
|
"loss": 0.12396180629730225,
|
|
"grad_norm": 1.4171065092086792,
|
|
"learning_rate": 1.0344827586206896e-08,
|
|
"num_tokens": 3148182.0,
|
|
"completions/mean_length": 814.625,
|
|
"completions/min_length": 181.0,
|
|
"completions/max_length": 929.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 733.75,
|
|
"completions/min_terminated_length": 181.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"tools/call_frequency": 18.625,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.18541666865348816,
|
|
"rewards/reward_func/std": 0.22297367453575134,
|
|
"reward": 0.18541666865348816,
|
|
"reward_std": 0.22297368943691254,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0008691527764312923,
|
|
"sampling/sampling_logp_difference/max": 0.31406283378601074,
|
|
"sampling/importance_sampling_ratio/min": 0.67269366979599,
|
|
"sampling/importance_sampling_ratio/mean": 0.8428615927696228,
|
|
"sampling/importance_sampling_ratio/max": 1.077307105064392,
|
|
"kl": 0.0003547998626345361,
|
|
"entropy": 0.021136773459147662,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.16849946975708,
|
|
"epoch": 0.007682291666666666,
|
|
"step": 295
|
|
},
|
|
{
|
|
"loss": 0.1688525676727295,
|
|
"grad_norm": 2.6168065071105957,
|
|
"learning_rate": 8.620689655172413e-09,
|
|
"num_tokens": 3159015.0,
|
|
"completions/mean_length": 668.75,
|
|
"completions/min_length": 243.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 543.4000244140625,
|
|
"completions/min_terminated_length": 243.0,
|
|
"completions/max_terminated_length": 932.0,
|
|
"tools/call_frequency": 16.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.3333333432674408,
|
|
"rewards/reward_func/std": 0.39651256799697876,
|
|
"reward": 0.3333333432674408,
|
|
"reward_std": 0.39651256799697876,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.002478270325809717,
|
|
"sampling/sampling_logp_difference/max": 0.4876605272293091,
|
|
"sampling/importance_sampling_ratio/min": 0.447879433631897,
|
|
"sampling/importance_sampling_ratio/mean": 0.8658635020256042,
|
|
"sampling/importance_sampling_ratio/max": 1.4510539770126343,
|
|
"kl": 0.001056952105500386,
|
|
"entropy": 0.04487227776553482,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.013084761798382,
|
|
"epoch": 0.0077083333333333335,
|
|
"step": 296
|
|
},
|
|
{
|
|
"loss": 0.39892399311065674,
|
|
"grad_norm": 2.8991482257843018,
|
|
"learning_rate": 6.896551724137931e-09,
|
|
"num_tokens": 3169254.0,
|
|
"completions/mean_length": 595.75,
|
|
"completions/min_length": 209.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 390.6000061035156,
|
|
"completions/min_terminated_length": 209.0,
|
|
"completions/max_terminated_length": 921.0,
|
|
"tools/call_frequency": 13.75,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.29720646142959595,
|
|
"rewards/reward_func/std": 0.2952474057674408,
|
|
"reward": 0.29720646142959595,
|
|
"reward_std": 0.2952474057674408,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0026496651116758585,
|
|
"sampling/sampling_logp_difference/max": 0.9173774719238281,
|
|
"sampling/importance_sampling_ratio/min": 0.3268800675868988,
|
|
"sampling/importance_sampling_ratio/mean": 1.015376091003418,
|
|
"sampling/importance_sampling_ratio/max": 1.3526206016540527,
|
|
"kl": 0.0012400131727190455,
|
|
"entropy": 0.043847846798598766,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 11.699225321412086,
|
|
"epoch": 0.007734375,
|
|
"step": 297
|
|
},
|
|
{
|
|
"loss": -0.21102824807167053,
|
|
"grad_norm": 1.6623992919921875,
|
|
"learning_rate": 5.172413793103448e-09,
|
|
"num_tokens": 3181309.0,
|
|
"completions/mean_length": 822.25,
|
|
"completions/min_length": 219.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/mean_terminated_length": 732.25,
|
|
"completions/min_terminated_length": 219.0,
|
|
"completions/max_terminated_length": 928.0,
|
|
"tools/call_frequency": 19.5,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.03125,
|
|
"rewards/reward_func/std": 0.0530330091714859,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.0530330091714859,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0013427839148789644,
|
|
"sampling/sampling_logp_difference/max": 0.6201149225234985,
|
|
"sampling/importance_sampling_ratio/min": 0.47519856691360474,
|
|
"sampling/importance_sampling_ratio/mean": 0.8805065155029297,
|
|
"sampling/importance_sampling_ratio/max": 1.2641569375991821,
|
|
"kl": 0.00035056304409408767,
|
|
"entropy": 0.02113774634199217,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 13.6938082780689,
|
|
"epoch": 0.007760416666666666,
|
|
"step": 298
|
|
},
|
|
{
|
|
"loss": -0.012455210089683533,
|
|
"grad_norm": 6.828906536102295,
|
|
"learning_rate": 3.4482758620689654e-09,
|
|
"num_tokens": 3192754.0,
|
|
"completions/mean_length": 743.625,
|
|
"completions/min_length": 206.0,
|
|
"completions/max_length": 935.0,
|
|
"completions/clipped_ratio": 0.625,
|
|
"completions/mean_terminated_length": 452.3333435058594,
|
|
"completions/min_terminated_length": 206.0,
|
|
"completions/max_terminated_length": 935.0,
|
|
"tools/call_frequency": 17.0,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.1302083432674408,
|
|
"rewards/reward_func/std": 0.25078994035720825,
|
|
"reward": 0.1302083432674408,
|
|
"reward_std": 0.25078994035720825,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.0015254232566803694,
|
|
"sampling/sampling_logp_difference/max": 0.9336514472961426,
|
|
"sampling/importance_sampling_ratio/min": 0.5875252485275269,
|
|
"sampling/importance_sampling_ratio/mean": 1.0684126615524292,
|
|
"sampling/importance_sampling_ratio/max": 2.433234214782715,
|
|
"kl": 0.0010421390925330343,
|
|
"entropy": 0.026804288267157972,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.708288345485926,
|
|
"epoch": 0.007786458333333334,
|
|
"step": 299
|
|
},
|
|
{
|
|
"loss": 0.13253474235534668,
|
|
"grad_norm": 2.2364230155944824,
|
|
"learning_rate": 1.7241379310344827e-09,
|
|
"num_tokens": 3203526.0,
|
|
"completions/mean_length": 661.625,
|
|
"completions/min_length": 200.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/clipped_ratio": 0.375,
|
|
"completions/mean_terminated_length": 511.20001220703125,
|
|
"completions/min_terminated_length": 200.0,
|
|
"completions/max_terminated_length": 932.0,
|
|
"tools/call_frequency": 15.375,
|
|
"tools/failure_frequency": 0.0,
|
|
"rewards/reward_func/mean": 0.277777761220932,
|
|
"rewards/reward_func/std": 0.37522774934768677,
|
|
"reward": 0.277777761220932,
|
|
"reward_std": 0.37522774934768677,
|
|
"frac_reward_zero_std": 0.0,
|
|
"sampling/sampling_logp_difference/mean": 0.001772936899214983,
|
|
"sampling/sampling_logp_difference/max": 0.37062156200408936,
|
|
"sampling/importance_sampling_ratio/min": 0.4278746247291565,
|
|
"sampling/importance_sampling_ratio/mean": 1.0974760055541992,
|
|
"sampling/importance_sampling_ratio/max": 1.4561865329742432,
|
|
"kl": 0.0013342537213247851,
|
|
"entropy": 0.03799662477103993,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"step_time": 12.152842845767736,
|
|
"epoch": 0.0078125,
|
|
"step": 300
|
|
},
|
|
{
|
|
"train_runtime": 4222.7288,
|
|
"train_samples_per_second": 0.568,
|
|
"train_steps_per_second": 0.071,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.05546886101374791,
|
|
"epoch": 0.0078125,
|
|
"step": 300
|
|
}
|
|
] |