1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"eval_steps": 50,
|
||
|
|
"global_step": 312,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.10751953125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1533.4,
|
||
|
|
"completions/mean_length": 467.5353515625,
|
||
|
|
"completions/mean_terminated_length": 338.8080322265625,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.016,
|
||
|
|
"grad_norm": 0.00392949627712369,
|
||
|
|
"learning_rate": 3.1249999999999997e-07,
|
||
|
|
"loss": 0.049,
|
||
|
|
"num_tokens": 19631594.0,
|
||
|
|
"reward": 0.2172173887491226,
|
||
|
|
"reward_std": 0.39361864924430845,
|
||
|
|
"rewards/accuracy_reward": 0.0654296875,
|
||
|
|
"rewards/brier_reward": 0.11202050149440765,
|
||
|
|
"rewards/confidence_one_or_zero": 0.4240234375,
|
||
|
|
"rewards/format_reward": 0.26181640625,
|
||
|
|
"rewards/mean_confidence_reward": 0.5485599517822266,
|
||
|
|
"rewards/smce": -0.4833765268325806,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.1005859375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1530.6,
|
||
|
|
"completions/mean_length": 442.7353515625,
|
||
|
|
"completions/mean_terminated_length": 320.555126953125,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.032,
|
||
|
|
"grad_norm": 0.0044568004086613655,
|
||
|
|
"learning_rate": 6.249999999999999e-07,
|
||
|
|
"loss": 0.0491,
|
||
|
|
"num_tokens": 39265556.0,
|
||
|
|
"reward": 0.2336795836687088,
|
||
|
|
"reward_std": 0.392722350358963,
|
||
|
|
"rewards/accuracy_reward": 0.06435546875,
|
||
|
|
"rewards/brier_reward": 0.1137367457151413,
|
||
|
|
"rewards/confidence_one_or_zero": 0.436328125,
|
||
|
|
"rewards/format_reward": 0.2943359375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5709773063659668,
|
||
|
|
"rewards/smce": -0.5070997059345246,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05361328125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1515.0,
|
||
|
|
"completions/mean_length": 311.44423828125,
|
||
|
|
"completions/mean_terminated_length": 242.5269287109375,
|
||
|
|
"completions/min_length": 1.6,
|
||
|
|
"completions/min_terminated_length": 1.6,
|
||
|
|
"epoch": 0.048,
|
||
|
|
"grad_norm": 0.0040215696208179,
|
||
|
|
"learning_rate": 9.374999999999999e-07,
|
||
|
|
"loss": 0.0516,
|
||
|
|
"num_tokens": 57503481.0,
|
||
|
|
"reward": 0.3996890068054199,
|
||
|
|
"reward_std": 0.43802899718284605,
|
||
|
|
"rewards/accuracy_reward": 0.1064453125,
|
||
|
|
"rewards/brier_reward": 0.1968934416770935,
|
||
|
|
"rewards/confidence_one_or_zero": 0.53427734375,
|
||
|
|
"rewards/format_reward": 0.501953125,
|
||
|
|
"rewards/mean_confidence_reward": 0.6975669860839844,
|
||
|
|
"rewards/smce": -0.5916327595710754,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0208984375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1477.2,
|
||
|
|
"completions/mean_length": 209.77099609375,
|
||
|
|
"completions/mean_terminated_length": 181.59221801757812,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"epoch": 0.064,
|
||
|
|
"grad_norm": 0.0028011025860905647,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0307,
|
||
|
|
"num_tokens": 74569936.0,
|
||
|
|
"reward": 0.6740051746368408,
|
||
|
|
"reward_std": 0.3955989480018616,
|
||
|
|
"rewards/accuracy_reward": 0.1849609375,
|
||
|
|
"rewards/brier_reward": 0.33165050745010377,
|
||
|
|
"rewards/confidence_one_or_zero": 0.5771484375,
|
||
|
|
"rewards/format_reward": 0.83759765625,
|
||
|
|
"rewards/mean_confidence_reward": 0.8032736539840698,
|
||
|
|
"rewards/smce": -0.6201528906822205,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.01220703125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1322.0,
|
||
|
|
"completions/mean_length": 160.39345703125,
|
||
|
|
"completions/mean_terminated_length": 143.40494079589843,
|
||
|
|
"completions/min_length": 20.4,
|
||
|
|
"completions/min_terminated_length": 20.4,
|
||
|
|
"epoch": 0.08,
|
||
|
|
"grad_norm": 0.0025906760711222887,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0177,
|
||
|
|
"num_tokens": 91145517.0,
|
||
|
|
"reward": 0.7873133897781373,
|
||
|
|
"reward_std": 0.32279191017150877,
|
||
|
|
"rewards/accuracy_reward": 0.2150390625,
|
||
|
|
"rewards/brier_reward": 0.4018263161182404,
|
||
|
|
"rewards/confidence_one_or_zero": 0.5064453125,
|
||
|
|
"rewards/format_reward": 0.96357421875,
|
||
|
|
"rewards/mean_confidence_reward": 0.7923147320747376,
|
||
|
|
"rewards/smce": -0.581539797782898,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00615234375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1209.2,
|
||
|
|
"completions/mean_length": 143.08671875,
|
||
|
|
"completions/mean_terminated_length": 134.46507263183594,
|
||
|
|
"completions/min_length": 29.0,
|
||
|
|
"completions/min_terminated_length": 29.0,
|
||
|
|
"epoch": 0.096,
|
||
|
|
"grad_norm": 0.0022409316152334213,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0088,
|
||
|
|
"num_tokens": 107655333.0,
|
||
|
|
"reward": 0.8789462089538574,
|
||
|
|
"reward_std": 0.28507915139198303,
|
||
|
|
"rewards/accuracy_reward": 0.24111328125,
|
||
|
|
"rewards/brier_reward": 0.5377443969249726,
|
||
|
|
"rewards/confidence_one_or_zero": 0.32646484375,
|
||
|
|
"rewards/format_reward": 0.983203125,
|
||
|
|
"rewards/mean_confidence_reward": 0.6468424916267395,
|
||
|
|
"rewards/smce": -0.4170338332653046,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.003515625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 959.0,
|
||
|
|
"completions/mean_length": 134.5802734375,
|
||
|
|
"completions/mean_terminated_length": 129.63559265136718,
|
||
|
|
"completions/min_length": 39.4,
|
||
|
|
"completions/min_terminated_length": 39.4,
|
||
|
|
"epoch": 0.112,
|
||
|
|
"grad_norm": 0.0014994166558608413,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0071,
|
||
|
|
"num_tokens": 124142907.0,
|
||
|
|
"reward": 0.9858169555664062,
|
||
|
|
"reward_std": 0.21316042244434358,
|
||
|
|
"rewards/accuracy_reward": 0.278515625,
|
||
|
|
"rewards/brier_reward": 0.7030979990959167,
|
||
|
|
"rewards/confidence_one_or_zero": 0.1814453125,
|
||
|
|
"rewards/format_reward": 0.991796875,
|
||
|
|
"rewards/mean_confidence_reward": 0.42606642842292786,
|
||
|
|
"rewards/smce": -0.17777334451675414,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0015625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 761.2,
|
||
|
|
"completions/mean_length": 130.01904296875,
|
||
|
|
"completions/mean_terminated_length": 127.81943054199219,
|
||
|
|
"completions/min_length": 43.2,
|
||
|
|
"completions/min_terminated_length": 43.2,
|
||
|
|
"epoch": 0.128,
|
||
|
|
"grad_norm": 0.0010288904886692762,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0027,
|
||
|
|
"num_tokens": 140390974.0,
|
||
|
|
"reward": 1.0206197261810304,
|
||
|
|
"reward_std": 0.15273716747760774,
|
||
|
|
"rewards/accuracy_reward": 0.2755859375,
|
||
|
|
"rewards/brier_reward": 0.7700915813446045,
|
||
|
|
"rewards/confidence_one_or_zero": 0.14150390625,
|
||
|
|
"rewards/format_reward": 0.99619140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3022731125354767,
|
||
|
|
"rewards/smce": -0.06302693337202073,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1236.8,
|
||
|
|
"completions/max_terminated_length": 817.4,
|
||
|
|
"completions/mean_length": 125.5994140625,
|
||
|
|
"completions/mean_terminated_length": 124.77249755859376,
|
||
|
|
"completions/min_length": 50.8,
|
||
|
|
"completions/min_terminated_length": 50.8,
|
||
|
|
"epoch": 0.144,
|
||
|
|
"grad_norm": 0.0009289440931752324,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"num_tokens": 156627544.0,
|
||
|
|
"reward": 1.0439924240112304,
|
||
|
|
"reward_std": 0.12395856976509094,
|
||
|
|
"rewards/accuracy_reward": 0.327734375,
|
||
|
|
"rewards/brier_reward": 0.7630051136016845,
|
||
|
|
"rewards/confidence_one_or_zero": 0.107421875,
|
||
|
|
"rewards/format_reward": 0.99814453125,
|
||
|
|
"rewards/mean_confidence_reward": 0.24927894175052642,
|
||
|
|
"rewards/smce": -0.08998571038246155,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00107421875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 507.2,
|
||
|
|
"completions/mean_length": 126.9736328125,
|
||
|
|
"completions/mean_terminated_length": 125.45684814453125,
|
||
|
|
"completions/min_length": 55.4,
|
||
|
|
"completions/min_terminated_length": 55.4,
|
||
|
|
"epoch": 0.16,
|
||
|
|
"grad_norm": 0.0008057853556238115,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0026,
|
||
|
|
"num_tokens": 172948682.0,
|
||
|
|
"reward": 1.0449287652969361,
|
||
|
|
"reward_std": 0.10962955206632614,
|
||
|
|
"rewards/accuracy_reward": 0.32119140625,
|
||
|
|
"rewards/brier_reward": 0.77177414894104,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0669921875,
|
||
|
|
"rewards/format_reward": 0.99765625,
|
||
|
|
"rewards/mean_confidence_reward": 0.2513626962900162,
|
||
|
|
"rewards/smce": -0.07648447304964065,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 255.5,
|
||
|
|
"eval_completions/max_terminated_length": 255.5,
|
||
|
|
"eval_completions/mean_length": 123.6274299621582,
|
||
|
|
"eval_completions/mean_terminated_length": 123.6274299621582,
|
||
|
|
"eval_completions/min_length": 58.0,
|
||
|
|
"eval_completions/min_terminated_length": 58.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 172948682.0,
|
||
|
|
"eval_reward": 1.021654725074768,
|
||
|
|
"eval_reward_std": 0.1542477086186409,
|
||
|
|
"eval_rewards/accuracy_reward": 0.23828125,
|
||
|
|
"eval_rewards/brier_reward": 0.8073956966400146,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.07421875,
|
||
|
|
"eval_rewards/format_reward": 0.998046875,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.2683268338441849,
|
||
|
|
"eval_rewards/smce": -0.04149917885661125,
|
||
|
|
"eval_runtime": 10.0732,
|
||
|
|
"eval_samples_per_second": 49.637,
|
||
|
|
"eval_steps_per_second": 0.199,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1093.0,
|
||
|
|
"completions/max_terminated_length": 615.6,
|
||
|
|
"completions/mean_length": 125.6130859375,
|
||
|
|
"completions/mean_terminated_length": 125.19995574951172,
|
||
|
|
"completions/min_length": 42.2,
|
||
|
|
"completions/min_terminated_length": 42.2,
|
||
|
|
"epoch": 0.176,
|
||
|
|
"grad_norm": 0.000889862421900034,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 189472080.0,
|
||
|
|
"reward": 1.045188045501709,
|
||
|
|
"reward_std": 0.10808687955141068,
|
||
|
|
"rewards/accuracy_reward": 0.313671875,
|
||
|
|
"rewards/brier_reward": 0.7784487485885621,
|
||
|
|
"rewards/confidence_one_or_zero": 0.037109375,
|
||
|
|
"rewards/format_reward": 0.99873046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.27597495913505554,
|
||
|
|
"rewards/smce": -0.04757043533027172,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1110.2,
|
||
|
|
"completions/max_terminated_length": 487.0,
|
||
|
|
"completions/mean_length": 125.22353515625,
|
||
|
|
"completions/mean_terminated_length": 124.67266235351562,
|
||
|
|
"completions/min_length": 56.8,
|
||
|
|
"completions/min_terminated_length": 56.8,
|
||
|
|
"epoch": 0.192,
|
||
|
|
"grad_norm": 0.0009008679771795869,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 205569185.0,
|
||
|
|
"reward": 1.061819863319397,
|
||
|
|
"reward_std": 0.11279134750366211,
|
||
|
|
"rewards/accuracy_reward": 0.35380859375,
|
||
|
|
"rewards/brier_reward": 0.7713178038597107,
|
||
|
|
"rewards/confidence_one_or_zero": 0.016015625,
|
||
|
|
"rewards/format_reward": 0.99912109375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3114722430706024,
|
||
|
|
"rewards/smce": -0.06084534525871277,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 739.6,
|
||
|
|
"completions/max_terminated_length": 579.0,
|
||
|
|
"completions/mean_length": 128.28876953125,
|
||
|
|
"completions/mean_terminated_length": 128.01407470703126,
|
||
|
|
"completions/min_length": 58.4,
|
||
|
|
"completions/min_terminated_length": 58.4,
|
||
|
|
"epoch": 0.208,
|
||
|
|
"grad_norm": 0.0007912637665867805,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 221915086.0,
|
||
|
|
"reward": 1.0683193922042846,
|
||
|
|
"reward_std": 0.11826074719429017,
|
||
|
|
"rewards/accuracy_reward": 0.37177734375,
|
||
|
|
"rewards/brier_reward": 0.7660857439041138,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0056640625,
|
||
|
|
"rewards/format_reward": 0.99912109375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3447410583496094,
|
||
|
|
"rewards/smce": -0.03460576832294464,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 341.8,
|
||
|
|
"completions/max_terminated_length": 341.8,
|
||
|
|
"completions/mean_length": 125.3783203125,
|
||
|
|
"completions/mean_terminated_length": 125.3783203125,
|
||
|
|
"completions/min_length": 58.8,
|
||
|
|
"completions/min_terminated_length": 58.8,
|
||
|
|
"epoch": 0.224,
|
||
|
|
"grad_norm": 0.0009502115426585078,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 238352144.0,
|
||
|
|
"reward": 1.068940782546997,
|
||
|
|
"reward_std": 0.11054961085319519,
|
||
|
|
"rewards/accuracy_reward": 0.3662109375,
|
||
|
|
"rewards/brier_reward": 0.7722442865371704,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0056640625,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.35045996904373167,
|
||
|
|
"rewards/smce": -0.03791281208395958,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 806.0,
|
||
|
|
"completions/max_terminated_length": 360.4,
|
||
|
|
"completions/mean_length": 125.35654296875,
|
||
|
|
"completions/mean_terminated_length": 125.08109130859376,
|
||
|
|
"completions/min_length": 55.4,
|
||
|
|
"completions/min_terminated_length": 55.4,
|
||
|
|
"epoch": 0.24,
|
||
|
|
"grad_norm": 0.0009807253954932094,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 254887475.0,
|
||
|
|
"reward": 1.0809075117111206,
|
||
|
|
"reward_std": 0.12458073198795319,
|
||
|
|
"rewards/accuracy_reward": 0.405078125,
|
||
|
|
"rewards/brier_reward": 0.7581897139549255,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00234375,
|
||
|
|
"rewards/format_reward": 0.99921875,
|
||
|
|
"rewards/mean_confidence_reward": 0.37911379933357237,
|
||
|
|
"rewards/smce": -0.06723080649971962,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 595.6,
|
||
|
|
"completions/max_terminated_length": 397.0,
|
||
|
|
"completions/mean_length": 123.59150390625,
|
||
|
|
"completions/mean_terminated_length": 123.45372619628907,
|
||
|
|
"completions/min_length": 57.8,
|
||
|
|
"completions/min_terminated_length": 57.8,
|
||
|
|
"epoch": 0.256,
|
||
|
|
"grad_norm": 0.0009870625799521804,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 271207868.0,
|
||
|
|
"reward": 1.07213237285614,
|
||
|
|
"reward_std": 0.1215924084186554,
|
||
|
|
"rewards/accuracy_reward": 0.37939453125,
|
||
|
|
"rewards/brier_reward": 0.7657799482345581,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0015625,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4004322052001953,
|
||
|
|
"rewards/smce": -0.06174885742366314,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1042.4,
|
||
|
|
"completions/max_terminated_length": 344.4,
|
||
|
|
"completions/mean_length": 126.02294921875,
|
||
|
|
"completions/mean_terminated_length": 125.4724334716797,
|
||
|
|
"completions/min_length": 60.4,
|
||
|
|
"completions/min_terminated_length": 60.4,
|
||
|
|
"epoch": 0.272,
|
||
|
|
"grad_norm": 0.0008904941496439278,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 287464039.0,
|
||
|
|
"reward": 1.0802291870117187,
|
||
|
|
"reward_std": 0.11962677985429764,
|
||
|
|
"rewards/accuracy_reward": 0.3966796875,
|
||
|
|
"rewards/brier_reward": 0.7651029467582703,
|
||
|
|
"rewards/confidence_one_or_zero": 0.001953125,
|
||
|
|
"rewards/format_reward": 0.9990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3965742290019989,
|
||
|
|
"rewards/smce": -0.0348443428054452,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1114.2,
|
||
|
|
"completions/max_terminated_length": 378.4,
|
||
|
|
"completions/mean_length": 123.46328125,
|
||
|
|
"completions/mean_terminated_length": 123.04900360107422,
|
||
|
|
"completions/min_length": 57.8,
|
||
|
|
"completions/min_terminated_length": 57.8,
|
||
|
|
"epoch": 0.288,
|
||
|
|
"grad_norm": 0.0010928181000053883,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 303686479.0,
|
||
|
|
"reward": 1.079901361465454,
|
||
|
|
"reward_std": 0.11479000747203827,
|
||
|
|
"rewards/accuracy_reward": 0.39150390625,
|
||
|
|
"rewards/brier_reward": 0.7692065358161926,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0017578125,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.39766603112220766,
|
||
|
|
"rewards/smce": -0.05178325399756432,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 347.8,
|
||
|
|
"completions/max_terminated_length": 347.8,
|
||
|
|
"completions/mean_length": 122.50859375,
|
||
|
|
"completions/mean_terminated_length": 122.50859375,
|
||
|
|
"completions/min_length": 58.0,
|
||
|
|
"completions/min_terminated_length": 58.0,
|
||
|
|
"epoch": 0.304,
|
||
|
|
"grad_norm": 0.0009639639756642282,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 319870919.0,
|
||
|
|
"reward": 1.087165641784668,
|
||
|
|
"reward_std": 0.11266087889671325,
|
||
|
|
"rewards/accuracy_reward": 0.40869140625,
|
||
|
|
"rewards/brier_reward": 0.7662365555763244,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0009765625,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3889293670654297,
|
||
|
|
"rewards/smce": -0.04021448343992233,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 836.0,
|
||
|
|
"completions/max_terminated_length": 372.6,
|
||
|
|
"completions/mean_length": 123.7615234375,
|
||
|
|
"completions/mean_terminated_length": 123.48605194091797,
|
||
|
|
"completions/min_length": 57.0,
|
||
|
|
"completions/min_terminated_length": 57.0,
|
||
|
|
"epoch": 0.32,
|
||
|
|
"grad_norm": 0.0008017583750188351,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 336226941.0,
|
||
|
|
"reward": 1.0964327812194825,
|
||
|
|
"reward_std": 0.10867121815681458,
|
||
|
|
"rewards/accuracy_reward": 0.42861328125,
|
||
|
|
"rewards/brier_reward": 0.7652536988258362,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.3929861426353455,
|
||
|
|
"rewards/smce": -0.07092485018074512,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 230.5,
|
||
|
|
"eval_completions/max_terminated_length": 230.5,
|
||
|
|
"eval_completions/mean_length": 123.81131362915039,
|
||
|
|
"eval_completions/mean_terminated_length": 123.81131362915039,
|
||
|
|
"eval_completions/min_length": 66.0,
|
||
|
|
"eval_completions/min_terminated_length": 66.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 336226941.0,
|
||
|
|
"eval_reward": 1.0621264576911926,
|
||
|
|
"eval_reward_std": 0.2113262265920639,
|
||
|
|
"eval_rewards/accuracy_reward": 0.337890625,
|
||
|
|
"eval_rewards/brier_reward": 0.7869812250137329,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.001953125,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.39699217677116394,
|
||
|
|
"eval_rewards/smce": -0.06198457069694996,
|
||
|
|
"eval_runtime": 9.5484,
|
||
|
|
"eval_samples_per_second": 52.365,
|
||
|
|
"eval_steps_per_second": 0.209,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 559.2,
|
||
|
|
"completions/max_terminated_length": 332.4,
|
||
|
|
"completions/mean_length": 122.5482421875,
|
||
|
|
"completions/mean_terminated_length": 122.41039276123047,
|
||
|
|
"completions/min_length": 58.4,
|
||
|
|
"completions/min_terminated_length": 58.4,
|
||
|
|
"epoch": 0.336,
|
||
|
|
"grad_norm": 0.000917735684197396,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 352204267.0,
|
||
|
|
"reward": 1.093878197669983,
|
||
|
|
"reward_std": 0.1106439396739006,
|
||
|
|
"rewards/accuracy_reward": 0.42294921875,
|
||
|
|
"rewards/brier_reward": 0.7657433271408081,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.40318635702133176,
|
||
|
|
"rewards/smce": -0.054635743424296376,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 554.6,
|
||
|
|
"completions/max_terminated_length": 328.8,
|
||
|
|
"completions/mean_length": 126.69482421875,
|
||
|
|
"completions/mean_terminated_length": 126.5573974609375,
|
||
|
|
"completions/min_length": 60.4,
|
||
|
|
"completions/min_terminated_length": 60.4,
|
||
|
|
"epoch": 0.352,
|
||
|
|
"grad_norm": 0.0008927257149480283,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 368762038.0,
|
||
|
|
"reward": 1.0763854026794433,
|
||
|
|
"reward_std": 0.10895265638828278,
|
||
|
|
"rewards/accuracy_reward": 0.38125,
|
||
|
|
"rewards/brier_reward": 0.7718688130378724,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4014307737350464,
|
||
|
|
"rewards/smce": -0.02511356510221958,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 517.6,
|
||
|
|
"completions/max_terminated_length": 517.6,
|
||
|
|
"completions/mean_length": 132.65068359375,
|
||
|
|
"completions/mean_terminated_length": 132.65068359375,
|
||
|
|
"completions/min_length": 63.6,
|
||
|
|
"completions/min_terminated_length": 63.6,
|
||
|
|
"epoch": 0.368,
|
||
|
|
"grad_norm": 0.000985518447123468,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0006,
|
||
|
|
"num_tokens": 385185853.0,
|
||
|
|
"reward": 1.0864951610565186,
|
||
|
|
"reward_std": 0.10731481909751892,
|
||
|
|
"rewards/accuracy_reward": 0.3982421875,
|
||
|
|
"rewards/brier_reward": 0.775156557559967,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4145812392234802,
|
||
|
|
"rewards/smce": -0.04092612899839878,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 592.8,
|
||
|
|
"completions/max_terminated_length": 356.0,
|
||
|
|
"completions/mean_length": 138.587890625,
|
||
|
|
"completions/mean_terminated_length": 138.31441650390624,
|
||
|
|
"completions/min_length": 58.6,
|
||
|
|
"completions/min_terminated_length": 58.6,
|
||
|
|
"epoch": 0.384,
|
||
|
|
"grad_norm": 0.0009577263845130801,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 401461505.0,
|
||
|
|
"reward": 1.097441053390503,
|
||
|
|
"reward_std": 0.11049059182405471,
|
||
|
|
"rewards/accuracy_reward": 0.41982421875,
|
||
|
|
"rewards/brier_reward": 0.7755703687667846,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4149980843067169,
|
||
|
|
"rewards/smce": -0.03180096782743931,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 444.4,
|
||
|
|
"completions/max_terminated_length": 444.4,
|
||
|
|
"completions/mean_length": 145.02041015625,
|
||
|
|
"completions/mean_terminated_length": 145.02041015625,
|
||
|
|
"completions/min_length": 65.2,
|
||
|
|
"completions/min_terminated_length": 65.2,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"grad_norm": 0.0009062528843060136,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 417982962.0,
|
||
|
|
"reward": 1.0899156332015991,
|
||
|
|
"reward_std": 0.11331143081188202,
|
||
|
|
"rewards/accuracy_reward": 0.39873046875,
|
||
|
|
"rewards/brier_reward": 0.7814009428024292,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0005859375,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.41860421299934386,
|
||
|
|
"rewards/smce": -0.030102025158703326,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 867.0,
|
||
|
|
"completions/max_terminated_length": 467.8,
|
||
|
|
"completions/mean_length": 153.2337890625,
|
||
|
|
"completions/mean_terminated_length": 152.82854614257812,
|
||
|
|
"completions/min_length": 65.4,
|
||
|
|
"completions/min_terminated_length": 65.4,
|
||
|
|
"epoch": 0.416,
|
||
|
|
"grad_norm": 0.0009052875684574246,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 434433260.0,
|
||
|
|
"reward": 1.0993643522262573,
|
||
|
|
"reward_std": 0.11355077922344207,
|
||
|
|
"rewards/accuracy_reward": 0.4205078125,
|
||
|
|
"rewards/brier_reward": 0.7787749409675598,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4114785134792328,
|
||
|
|
"rewards/smce": -0.026191336289048194,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1097.6,
|
||
|
|
"completions/max_terminated_length": 478.6,
|
||
|
|
"completions/mean_length": 158.43994140625,
|
||
|
|
"completions/mean_terminated_length": 158.03628845214843,
|
||
|
|
"completions/min_length": 69.6,
|
||
|
|
"completions/min_terminated_length": 69.6,
|
||
|
|
"epoch": 0.432,
|
||
|
|
"grad_norm": 0.0008921238477341831,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 451070021.0,
|
||
|
|
"reward": 1.1083012580871583,
|
||
|
|
"reward_std": 0.10892681926488876,
|
||
|
|
"rewards/accuracy_reward": 0.44169921875,
|
||
|
|
"rewards/brier_reward": 0.7755492091178894,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4254137694835663,
|
||
|
|
"rewards/smce": -0.03537834882736206,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 857.0,
|
||
|
|
"completions/max_terminated_length": 403.8,
|
||
|
|
"completions/mean_length": 162.83359375,
|
||
|
|
"completions/mean_terminated_length": 162.4312957763672,
|
||
|
|
"completions/min_length": 64.2,
|
||
|
|
"completions/min_terminated_length": 64.2,
|
||
|
|
"epoch": 0.448,
|
||
|
|
"grad_norm": 0.0008144128369167447,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 467690237.0,
|
||
|
|
"reward": 1.0995479583740235,
|
||
|
|
"reward_std": 0.11465191096067429,
|
||
|
|
"rewards/accuracy_reward": 0.42607421875,
|
||
|
|
"rewards/brier_reward": 0.7738674759864808,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.44896326065063474,
|
||
|
|
"rewards/smce": -0.0358337426558137,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1316.2,
|
||
|
|
"completions/max_terminated_length": 476.2,
|
||
|
|
"completions/mean_length": 164.81923828125,
|
||
|
|
"completions/mean_terminated_length": 164.14944458007812,
|
||
|
|
"completions/min_length": 75.0,
|
||
|
|
"completions/min_terminated_length": 75.0,
|
||
|
|
"epoch": 0.464,
|
||
|
|
"grad_norm": 0.0008099125698208809,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"num_tokens": 484548802.0,
|
||
|
|
"reward": 1.0764792203903197,
|
||
|
|
"reward_std": 0.11227991878986358,
|
||
|
|
"rewards/accuracy_reward": 0.37109375,
|
||
|
|
"rewards/brier_reward": 0.7830622434616089,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4407523512840271,
|
||
|
|
"rewards/smce": -0.0710053825750947,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 665.4,
|
||
|
|
"completions/max_terminated_length": 441.8,
|
||
|
|
"completions/mean_length": 162.240625,
|
||
|
|
"completions/mean_terminated_length": 162.10667419433594,
|
||
|
|
"completions/min_length": 76.8,
|
||
|
|
"completions/min_terminated_length": 76.8,
|
||
|
|
"epoch": 0.48,
|
||
|
|
"grad_norm": 0.000975205737631768,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 501258178.0,
|
||
|
|
"reward": 1.1072642087936402,
|
||
|
|
"reward_std": 0.10939528942108154,
|
||
|
|
"rewards/accuracy_reward": 0.44150390625,
|
||
|
|
"rewards/brier_reward": 0.7736056089401245,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4199238717556,
|
||
|
|
"rewards/smce": -0.04843057133257389,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 317.5,
|
||
|
|
"eval_completions/max_terminated_length": 317.5,
|
||
|
|
"eval_completions/mean_length": 163.09646606445312,
|
||
|
|
"eval_completions/mean_terminated_length": 163.09646606445312,
|
||
|
|
"eval_completions/min_length": 79.0,
|
||
|
|
"eval_completions/min_terminated_length": 79.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 501258178.0,
|
||
|
|
"eval_reward": 1.073753833770752,
|
||
|
|
"eval_reward_std": 0.21566922962665558,
|
||
|
|
"eval_rewards/accuracy_reward": 0.35546875,
|
||
|
|
"eval_rewards/brier_reward": 0.7925025522708893,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.40083983540534973,
|
||
|
|
"eval_rewards/smce": -0.04645352065563202,
|
||
|
|
"eval_runtime": 11.8477,
|
||
|
|
"eval_samples_per_second": 42.202,
|
||
|
|
"eval_steps_per_second": 0.169,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 678.4,
|
||
|
|
"completions/max_terminated_length": 462.0,
|
||
|
|
"completions/mean_length": 160.70107421875,
|
||
|
|
"completions/mean_terminated_length": 160.56687927246094,
|
||
|
|
"completions/min_length": 72.2,
|
||
|
|
"completions/min_terminated_length": 72.2,
|
||
|
|
"epoch": 0.496,
|
||
|
|
"grad_norm": 0.0009712612954899669,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 518211597.0,
|
||
|
|
"reward": 1.1064111471176148,
|
||
|
|
"reward_std": 0.10755210518836975,
|
||
|
|
"rewards/accuracy_reward": 0.44091796875,
|
||
|
|
"rewards/brier_reward": 0.7725687384605407,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.408827143907547,
|
||
|
|
"rewards/smce": -0.037215230613946916,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 699.4,
|
||
|
|
"completions/max_terminated_length": 468.8,
|
||
|
|
"completions/mean_length": 160.16220703125,
|
||
|
|
"completions/mean_terminated_length": 159.8941864013672,
|
||
|
|
"completions/min_length": 73.0,
|
||
|
|
"completions/min_terminated_length": 73.0,
|
||
|
|
"epoch": 0.512,
|
||
|
|
"grad_norm": 0.0009234827593900263,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 534997322.0,
|
||
|
|
"reward": 1.1220848321914674,
|
||
|
|
"reward_std": 0.10917693674564362,
|
||
|
|
"rewards/accuracy_reward": 0.4650390625,
|
||
|
|
"rewards/brier_reward": 0.7797863245010376,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.42907978892326354,
|
||
|
|
"rewards/smce": -0.04612938240170479,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 487.0,
|
||
|
|
"completions/max_terminated_length": 487.0,
|
||
|
|
"completions/mean_length": 164.23193359375,
|
||
|
|
"completions/mean_terminated_length": 164.23193359375,
|
||
|
|
"completions/min_length": 78.0,
|
||
|
|
"completions/min_terminated_length": 78.0,
|
||
|
|
"epoch": 0.528,
|
||
|
|
"grad_norm": 0.0009526491048745811,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 551708593.0,
|
||
|
|
"reward": 1.113792848587036,
|
||
|
|
"reward_std": 0.11661270409822463,
|
||
|
|
"rewards/accuracy_reward": 0.4375,
|
||
|
|
"rewards/brier_reward": 0.7903852462768555,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4550130248069763,
|
||
|
|
"rewards/smce": -0.030036850459873678,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 652.8,
|
||
|
|
"completions/max_terminated_length": 450.2,
|
||
|
|
"completions/mean_length": 162.72724609375,
|
||
|
|
"completions/mean_terminated_length": 162.5930206298828,
|
||
|
|
"completions/min_length": 71.0,
|
||
|
|
"completions/min_terminated_length": 71.0,
|
||
|
|
"epoch": 0.544,
|
||
|
|
"grad_norm": 0.0009181004716083407,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 568538504.0,
|
||
|
|
"reward": 1.1319088459014892,
|
||
|
|
"reward_std": 0.12556345760822296,
|
||
|
|
"rewards/accuracy_reward": 0.490234375,
|
||
|
|
"rewards/brier_reward": 0.7738756775856018,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.48188769817352295,
|
||
|
|
"rewards/smce": -0.019567849487066268,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 440.4,
|
||
|
|
"completions/max_terminated_length": 440.4,
|
||
|
|
"completions/mean_length": 158.8845703125,
|
||
|
|
"completions/mean_terminated_length": 158.8845703125,
|
||
|
|
"completions/min_length": 70.4,
|
||
|
|
"completions/min_terminated_length": 70.4,
|
||
|
|
"epoch": 0.56,
|
||
|
|
"grad_norm": 0.0008448117878288031,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 584986890.0,
|
||
|
|
"reward": 1.1044183254241944,
|
||
|
|
"reward_std": 0.11335271000862121,
|
||
|
|
"rewards/accuracy_reward": 0.419921875,
|
||
|
|
"rewards/brier_reward": 0.7894919633865356,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.476855480670929,
|
||
|
|
"rewards/smce": -0.057809342816472056,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 412.2,
|
||
|
|
"completions/max_terminated_length": 412.2,
|
||
|
|
"completions/mean_length": 158.0970703125,
|
||
|
|
"completions/mean_terminated_length": 158.0970703125,
|
||
|
|
"completions/min_length": 73.6,
|
||
|
|
"completions/min_terminated_length": 73.6,
|
||
|
|
"epoch": 0.576,
|
||
|
|
"grad_norm": 0.0007008586544543505,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 601792428.0,
|
||
|
|
"reward": 1.1063513040542603,
|
||
|
|
"reward_std": 0.10015368312597275,
|
||
|
|
"rewards/accuracy_reward": 0.42802734375,
|
||
|
|
"rewards/brier_reward": 0.7851521253585816,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4443991184234619,
|
||
|
|
"rewards/smce": -0.047779816389083865,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 655.8,
|
||
|
|
"completions/max_terminated_length": 417.0,
|
||
|
|
"completions/mean_length": 154.9142578125,
|
||
|
|
"completions/mean_terminated_length": 154.64378356933594,
|
||
|
|
"completions/min_length": 74.6,
|
||
|
|
"completions/min_terminated_length": 74.6,
|
||
|
|
"epoch": 0.592,
|
||
|
|
"grad_norm": 0.0007284689345397055,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 618546462.0,
|
||
|
|
"reward": 1.1119030475616456,
|
||
|
|
"reward_std": 0.09350670576095581,
|
||
|
|
"rewards/accuracy_reward": 0.44013671875,
|
||
|
|
"rewards/brier_reward": 0.7842747569084167,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4142773449420929,
|
||
|
|
"rewards/smce": -0.03132548592984676,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 424.8,
|
||
|
|
"completions/max_terminated_length": 424.8,
|
||
|
|
"completions/mean_length": 154.5126953125,
|
||
|
|
"completions/mean_terminated_length": 154.5126953125,
|
||
|
|
"completions/min_length": 74.4,
|
||
|
|
"completions/min_terminated_length": 74.4,
|
||
|
|
"epoch": 0.608,
|
||
|
|
"grad_norm": 0.0008701604674570262,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 635128160.0,
|
||
|
|
"reward": 1.1218822956085206,
|
||
|
|
"reward_std": 0.10320815443992615,
|
||
|
|
"rewards/accuracy_reward": 0.44443359375,
|
||
|
|
"rewards/brier_reward": 0.7996853113174438,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.43260841369628905,
|
||
|
|
"rewards/smce": -0.035519332997500895,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 618.2,
|
||
|
|
"completions/max_terminated_length": 392.8,
|
||
|
|
"completions/mean_length": 157.1732421875,
|
||
|
|
"completions/mean_terminated_length": 157.0383514404297,
|
||
|
|
"completions/min_length": 69.4,
|
||
|
|
"completions/min_terminated_length": 69.4,
|
||
|
|
"epoch": 0.624,
|
||
|
|
"grad_norm": 0.0011468639131635427,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 652081518.0,
|
||
|
|
"reward": 1.1084702968597413,
|
||
|
|
"reward_std": 0.1136349618434906,
|
||
|
|
"rewards/accuracy_reward": 0.4287109375,
|
||
|
|
"rewards/brier_reward": 0.7887693285942078,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4634462893009186,
|
||
|
|
"rewards/smce": -0.044282649829983714,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1093.6,
|
||
|
|
"completions/max_terminated_length": 420.0,
|
||
|
|
"completions/mean_length": 157.331640625,
|
||
|
|
"completions/mean_terminated_length": 156.92776794433593,
|
||
|
|
"completions/min_length": 74.6,
|
||
|
|
"completions/min_terminated_length": 74.6,
|
||
|
|
"epoch": 0.64,
|
||
|
|
"grad_norm": 0.0008401005179621279,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 669035282.0,
|
||
|
|
"reward": 1.143984031677246,
|
||
|
|
"reward_std": 0.11052749902009965,
|
||
|
|
"rewards/accuracy_reward": 0.49716796875,
|
||
|
|
"rewards/brier_reward": 0.7917260885238647,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4753476619720459,
|
||
|
|
"rewards/smce": -0.05361993685364723,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 320.0,
|
||
|
|
"eval_completions/max_terminated_length": 320.0,
|
||
|
|
"eval_completions/mean_length": 161.03005981445312,
|
||
|
|
"eval_completions/mean_terminated_length": 161.03005981445312,
|
||
|
|
"eval_completions/min_length": 83.5,
|
||
|
|
"eval_completions/min_terminated_length": 83.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 669035282.0,
|
||
|
|
"eval_reward": 1.0816600322723389,
|
||
|
|
"eval_reward_std": 0.2524609863758087,
|
||
|
|
"eval_rewards/accuracy_reward": 0.3671875,
|
||
|
|
"eval_rewards/brier_reward": 0.7971554696559906,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.46910156309604645,
|
||
|
|
"eval_rewards/smce": -0.10237480700016022,
|
||
|
|
"eval_runtime": 12.0043,
|
||
|
|
"eval_samples_per_second": 41.652,
|
||
|
|
"eval_steps_per_second": 0.167,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 623.4,
|
||
|
|
"completions/max_terminated_length": 380.4,
|
||
|
|
"completions/mean_length": 157.94111328125,
|
||
|
|
"completions/mean_terminated_length": 157.80642395019532,
|
||
|
|
"completions/min_length": 75.6,
|
||
|
|
"completions/min_terminated_length": 75.6,
|
||
|
|
"epoch": 0.656,
|
||
|
|
"grad_norm": 0.001043739845044911,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 685509143.0,
|
||
|
|
"reward": 1.1010662317276,
|
||
|
|
"reward_std": 0.11220352202653885,
|
||
|
|
"rewards/accuracy_reward": 0.417578125,
|
||
|
|
"rewards/brier_reward": 0.7853797674179077,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.47954005002975464,
|
||
|
|
"rewards/smce": -0.0630951227620244,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 612.2,
|
||
|
|
"completions/max_terminated_length": 395.2,
|
||
|
|
"completions/mean_length": 158.19736328125,
|
||
|
|
"completions/mean_terminated_length": 158.0628631591797,
|
||
|
|
"completions/min_length": 73.2,
|
||
|
|
"completions/min_terminated_length": 73.2,
|
||
|
|
"epoch": 0.672,
|
||
|
|
"grad_norm": 0.0007125500706024468,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 702042524.0,
|
||
|
|
"reward": 1.1255898952484131,
|
||
|
|
"reward_std": 0.09300058484077453,
|
||
|
|
"rewards/accuracy_reward": 0.45703125,
|
||
|
|
"rewards/brier_reward": 0.7946899771690369,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4336543083190918,
|
||
|
|
"rewards/smce": -0.04446049779653549,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 434.0,
|
||
|
|
"completions/max_terminated_length": 434.0,
|
||
|
|
"completions/mean_length": 158.30068359375,
|
||
|
|
"completions/mean_terminated_length": 158.30068359375,
|
||
|
|
"completions/min_length": 74.0,
|
||
|
|
"completions/min_terminated_length": 74.0,
|
||
|
|
"epoch": 0.688,
|
||
|
|
"grad_norm": 0.0009905676124617457,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 718617443.0,
|
||
|
|
"reward": 1.1272786140441895,
|
||
|
|
"reward_std": 0.09988767951726914,
|
||
|
|
"rewards/accuracy_reward": 0.45419921875,
|
||
|
|
"rewards/brier_reward": 0.8008356928825379,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.42675782442092897,
|
||
|
|
"rewards/smce": -0.04784578718245029,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 399.4,
|
||
|
|
"completions/max_terminated_length": 399.4,
|
||
|
|
"completions/mean_length": 160.6234375,
|
||
|
|
"completions/mean_terminated_length": 160.6234375,
|
||
|
|
"completions/min_length": 75.4,
|
||
|
|
"completions/min_terminated_length": 75.4,
|
||
|
|
"epoch": 0.704,
|
||
|
|
"grad_norm": 0.0008583047892898321,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 735128371.0,
|
||
|
|
"reward": 1.1382115125656127,
|
||
|
|
"reward_std": 0.1009350448846817,
|
||
|
|
"rewards/accuracy_reward": 0.48193359375,
|
||
|
|
"rewards/brier_reward": 0.7947717547416687,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.46465528607368467,
|
||
|
|
"rewards/smce": -0.028317653015255927,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 879.0,
|
||
|
|
"completions/max_terminated_length": 483.4,
|
||
|
|
"completions/mean_length": 164.634375,
|
||
|
|
"completions/mean_terminated_length": 164.36683349609376,
|
||
|
|
"completions/min_length": 64.8,
|
||
|
|
"completions/min_terminated_length": 64.8,
|
||
|
|
"epoch": 0.72,
|
||
|
|
"grad_norm": 0.0009704248514026403,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 751824083.0,
|
||
|
|
"reward": 1.1468546390533447,
|
||
|
|
"reward_std": 0.10621728748083115,
|
||
|
|
"rewards/accuracy_reward": 0.4880859375,
|
||
|
|
"rewards/brier_reward": 0.8062491059303284,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5047011613845825,
|
||
|
|
"rewards/smce": -0.033376603946089746,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 858.4,
|
||
|
|
"completions/max_terminated_length": 529.8,
|
||
|
|
"completions/mean_length": 165.065625,
|
||
|
|
"completions/mean_terminated_length": 164.79816284179688,
|
||
|
|
"completions/min_length": 77.0,
|
||
|
|
"completions/min_terminated_length": 77.0,
|
||
|
|
"epoch": 0.736,
|
||
|
|
"grad_norm": 0.0009971307590603828,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 768453939.0,
|
||
|
|
"reward": 1.1445934057235718,
|
||
|
|
"reward_std": 0.10860948264598846,
|
||
|
|
"rewards/accuracy_reward": 0.49833984375,
|
||
|
|
"rewards/brier_reward": 0.7916695952415467,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5039228439331055,
|
||
|
|
"rewards/smce": -0.053068242594599725,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 449.0,
|
||
|
|
"completions/max_terminated_length": 449.0,
|
||
|
|
"completions/mean_length": 169.20380859375,
|
||
|
|
"completions/mean_terminated_length": 169.20380859375,
|
||
|
|
"completions/min_length": 80.2,
|
||
|
|
"completions/min_terminated_length": 80.2,
|
||
|
|
"epoch": 0.752,
|
||
|
|
"grad_norm": 0.0009284823318012059,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 785413786.0,
|
||
|
|
"reward": 1.1324732303619385,
|
||
|
|
"reward_std": 0.10707245618104935,
|
||
|
|
"rewards/accuracy_reward": 0.47109375,
|
||
|
|
"rewards/brier_reward": 0.7942162990570069,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4771093845367432,
|
||
|
|
"rewards/smce": -0.036446777358651164,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 874.0,
|
||
|
|
"completions/max_terminated_length": 661.4,
|
||
|
|
"completions/mean_length": 175.115625,
|
||
|
|
"completions/mean_terminated_length": 174.98184814453126,
|
||
|
|
"completions/min_length": 74.6,
|
||
|
|
"completions/min_terminated_length": 74.6,
|
||
|
|
"epoch": 0.768,
|
||
|
|
"grad_norm": 0.0008781220531091094,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 802139674.0,
|
||
|
|
"reward": 1.1389093160629273,
|
||
|
|
"reward_std": 0.09807263612747193,
|
||
|
|
"rewards/accuracy_reward": 0.473046875,
|
||
|
|
"rewards/brier_reward": 0.8051334381103515,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.46835254430770873,
|
||
|
|
"rewards/smce": -0.016733316145837306,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 418.6,
|
||
|
|
"completions/max_terminated_length": 418.6,
|
||
|
|
"completions/mean_length": 171.97890625,
|
||
|
|
"completions/mean_terminated_length": 171.97890625,
|
||
|
|
"completions/min_length": 84.4,
|
||
|
|
"completions/min_terminated_length": 84.4,
|
||
|
|
"epoch": 0.784,
|
||
|
|
"grad_norm": 0.0009764548158273101,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 819075106.0,
|
||
|
|
"reward": 1.1391575574874877,
|
||
|
|
"reward_std": 0.10245644003152847,
|
||
|
|
"rewards/accuracy_reward": 0.48662109375,
|
||
|
|
"rewards/brier_reward": 0.7924958348274231,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.47624804377555846,
|
||
|
|
"rewards/smce": -0.08027197942137718,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 485.6,
|
||
|
|
"completions/max_terminated_length": 485.6,
|
||
|
|
"completions/mean_length": 169.80869140625,
|
||
|
|
"completions/mean_terminated_length": 169.80869140625,
|
||
|
|
"completions/min_length": 84.6,
|
||
|
|
"completions/min_terminated_length": 84.6,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 0.0011034056078642607,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 835824507.0,
|
||
|
|
"reward": 1.1558006286621094,
|
||
|
|
"reward_std": 0.11075237542390823,
|
||
|
|
"rewards/accuracy_reward": 0.50732421875,
|
||
|
|
"rewards/brier_reward": 0.8045778751373291,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.5076025426387787,
|
||
|
|
"rewards/smce": -0.030179687216877936,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 340.5,
|
||
|
|
"eval_completions/max_terminated_length": 340.5,
|
||
|
|
"eval_completions/mean_length": 174.92011260986328,
|
||
|
|
"eval_completions/mean_terminated_length": 174.92011260986328,
|
||
|
|
"eval_completions/min_length": 89.5,
|
||
|
|
"eval_completions/min_terminated_length": 89.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 835824507.0,
|
||
|
|
"eval_reward": 1.0990996360778809,
|
||
|
|
"eval_reward_std": 0.2727942317724228,
|
||
|
|
"eval_rewards/accuracy_reward": 0.40234375,
|
||
|
|
"eval_rewards/brier_reward": 0.7968310713768005,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.49873048067092896,
|
||
|
|
"eval_rewards/smce": -0.09765136614441872,
|
||
|
|
"eval_runtime": 11.7282,
|
||
|
|
"eval_samples_per_second": 42.632,
|
||
|
|
"eval_steps_per_second": 0.171,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 686.8,
|
||
|
|
"completions/max_terminated_length": 486.0,
|
||
|
|
"completions/mean_length": 171.3958984375,
|
||
|
|
"completions/mean_terminated_length": 171.12869567871093,
|
||
|
|
"completions/min_length": 88.0,
|
||
|
|
"completions/min_terminated_length": 88.0,
|
||
|
|
"epoch": 0.816,
|
||
|
|
"grad_norm": 0.0011392795713618398,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 852678769.0,
|
||
|
|
"reward": 1.1497153043746948,
|
||
|
|
"reward_std": 0.114381605386734,
|
||
|
|
"rewards/accuracy_reward": 0.494921875,
|
||
|
|
"rewards/brier_reward": 0.8050509095191956,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5240488409996032,
|
||
|
|
"rewards/smce": -0.03478394001722336,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 498.0,
|
||
|
|
"completions/max_terminated_length": 498.0,
|
||
|
|
"completions/mean_length": 176.65927734375,
|
||
|
|
"completions/mean_terminated_length": 176.65927734375,
|
||
|
|
"completions/min_length": 86.6,
|
||
|
|
"completions/min_terminated_length": 86.6,
|
||
|
|
"epoch": 0.832,
|
||
|
|
"grad_norm": 0.001069263438694179,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 869496112.0,
|
||
|
|
"reward": 1.1582709312438966,
|
||
|
|
"reward_std": 0.10491050034761429,
|
||
|
|
"rewards/accuracy_reward": 0.50341796875,
|
||
|
|
"rewards/brier_reward": 0.8133532643318176,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.5025537252426148,
|
||
|
|
"rewards/smce": -0.023029053397476674,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 502.4,
|
||
|
|
"completions/max_terminated_length": 502.4,
|
||
|
|
"completions/mean_length": 176.24482421875,
|
||
|
|
"completions/mean_terminated_length": 176.24482421875,
|
||
|
|
"completions/min_length": 85.4,
|
||
|
|
"completions/min_terminated_length": 85.4,
|
||
|
|
"epoch": 0.848,
|
||
|
|
"grad_norm": 0.0009353526984341443,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 886315227.0,
|
||
|
|
"reward": 1.139298939704895,
|
||
|
|
"reward_std": 0.10831455588340759,
|
||
|
|
"rewards/accuracy_reward": 0.47412109375,
|
||
|
|
"rewards/brier_reward": 0.804800283908844,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4926269590854645,
|
||
|
|
"rewards/smce": -0.03244311548769474,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 494.6,
|
||
|
|
"completions/max_terminated_length": 494.6,
|
||
|
|
"completions/mean_length": 179.4521484375,
|
||
|
|
"completions/mean_terminated_length": 179.4521484375,
|
||
|
|
"completions/min_length": 89.4,
|
||
|
|
"completions/min_terminated_length": 89.4,
|
||
|
|
"epoch": 0.864,
|
||
|
|
"grad_norm": 0.0009035305120050907,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 903139633.0,
|
||
|
|
"reward": 1.1543693780899047,
|
||
|
|
"reward_std": 0.10487027764320374,
|
||
|
|
"rewards/accuracy_reward": 0.50732421875,
|
||
|
|
"rewards/brier_reward": 0.8016931176185608,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.49616700410842896,
|
||
|
|
"rewards/smce": -0.02795556578785181,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1336.4,
|
||
|
|
"completions/max_terminated_length": 506.0,
|
||
|
|
"completions/mean_length": 182.682421875,
|
||
|
|
"completions/mean_terminated_length": 182.15309143066406,
|
||
|
|
"completions/min_length": 91.8,
|
||
|
|
"completions/min_terminated_length": 91.8,
|
||
|
|
"epoch": 0.88,
|
||
|
|
"grad_norm": 0.0007824220811016858,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 920157373.0,
|
||
|
|
"reward": 1.1384138345718384,
|
||
|
|
"reward_std": 0.09232062548398971,
|
||
|
|
"rewards/accuracy_reward": 0.46337890625,
|
||
|
|
"rewards/brier_reward": 0.8143354654312134,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4651464939117432,
|
||
|
|
"rewards/smce": -0.04969409257173538,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 473.0,
|
||
|
|
"completions/max_terminated_length": 473.0,
|
||
|
|
"completions/mean_length": 183.71357421875,
|
||
|
|
"completions/mean_terminated_length": 183.71357421875,
|
||
|
|
"completions/min_length": 92.2,
|
||
|
|
"completions/min_terminated_length": 92.2,
|
||
|
|
"epoch": 0.896,
|
||
|
|
"grad_norm": 0.0010242619318887591,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 937149448.0,
|
||
|
|
"reward": 1.143228530883789,
|
||
|
|
"reward_std": 0.09831419289112091,
|
||
|
|
"rewards/accuracy_reward": 0.47509765625,
|
||
|
|
"rewards/brier_reward": 0.8117444038391113,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4801484525203705,
|
||
|
|
"rewards/smce": -0.03859155103564262,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 474.8,
|
||
|
|
"completions/max_terminated_length": 474.8,
|
||
|
|
"completions/mean_length": 181.05634765625,
|
||
|
|
"completions/mean_terminated_length": 181.05634765625,
|
||
|
|
"completions/min_length": 88.0,
|
||
|
|
"completions/min_terminated_length": 88.0,
|
||
|
|
"epoch": 0.912,
|
||
|
|
"grad_norm": 0.0009002075530588627,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 954054761.0,
|
||
|
|
"reward": 1.1372110366821289,
|
||
|
|
"reward_std": 0.10027643889188767,
|
||
|
|
"rewards/accuracy_reward": 0.46640625,
|
||
|
|
"rewards/brier_reward": 0.8085625290870666,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.49353516697883604,
|
||
|
|
"rewards/smce": -0.05476636029779911,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 532.6,
|
||
|
|
"completions/max_terminated_length": 532.6,
|
||
|
|
"completions/mean_length": 174.0798828125,
|
||
|
|
"completions/mean_terminated_length": 174.0798828125,
|
||
|
|
"completions/min_length": 90.2,
|
||
|
|
"completions/min_terminated_length": 90.2,
|
||
|
|
"epoch": 0.928,
|
||
|
|
"grad_norm": 0.0008942595450207591,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 970864155.0,
|
||
|
|
"reward": 1.1491928815841674,
|
||
|
|
"reward_std": 0.09633661955595016,
|
||
|
|
"rewards/accuracy_reward": 0.50048828125,
|
||
|
|
"rewards/brier_reward": 0.7983452439308166,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4858271420001984,
|
||
|
|
"rewards/smce": -0.044872155785560607,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 682.2,
|
||
|
|
"completions/max_terminated_length": 449.4,
|
||
|
|
"completions/mean_length": 171.64189453125,
|
||
|
|
"completions/mean_terminated_length": 171.50814208984374,
|
||
|
|
"completions/min_length": 85.8,
|
||
|
|
"completions/min_terminated_length": 85.8,
|
||
|
|
"epoch": 0.944,
|
||
|
|
"grad_norm": 0.0009355693473480642,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 987597192.0,
|
||
|
|
"reward": 1.1451186180114745,
|
||
|
|
"reward_std": 0.0981968179345131,
|
||
|
|
"rewards/accuracy_reward": 0.4908203125,
|
||
|
|
"rewards/brier_reward": 0.8003330111503602,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.47065430879592896,
|
||
|
|
"rewards/smce": -0.08194140195846558,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 506.2,
|
||
|
|
"completions/max_terminated_length": 506.2,
|
||
|
|
"completions/mean_length": 172.3828125,
|
||
|
|
"completions/mean_terminated_length": 172.3828125,
|
||
|
|
"completions/min_length": 86.6,
|
||
|
|
"completions/min_terminated_length": 86.6,
|
||
|
|
"epoch": 0.96,
|
||
|
|
"grad_norm": 0.0008330916170962155,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 1004302712.0,
|
||
|
|
"reward": 1.1374672889709472,
|
||
|
|
"reward_std": 0.08538994491100312,
|
||
|
|
"rewards/accuracy_reward": 0.46298828125,
|
||
|
|
"rewards/brier_reward": 0.8122195124626159,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.46052245497703553,
|
||
|
|
"rewards/smce": -0.027407228574156763,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.96,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 349.5,
|
||
|
|
"eval_completions/max_terminated_length": 349.5,
|
||
|
|
"eval_completions/mean_length": 173.41207122802734,
|
||
|
|
"eval_completions/mean_terminated_length": 173.41207122802734,
|
||
|
|
"eval_completions/min_length": 98.0,
|
||
|
|
"eval_completions/min_terminated_length": 98.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 1004302712.0,
|
||
|
|
"eval_reward": 1.1020442247390747,
|
||
|
|
"eval_reward_std": 0.2523525282740593,
|
||
|
|
"eval_rewards/accuracy_reward": 0.388671875,
|
||
|
|
"eval_rewards/brier_reward": 0.8159765899181366,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4429687410593033,
|
||
|
|
"eval_rewards/smce": -0.05609375610947609,
|
||
|
|
"eval_runtime": 12.0381,
|
||
|
|
"eval_samples_per_second": 41.535,
|
||
|
|
"eval_steps_per_second": 0.166,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 707.2,
|
||
|
|
"completions/max_terminated_length": 491.8,
|
||
|
|
"completions/mean_length": 172.77578125,
|
||
|
|
"completions/mean_terminated_length": 172.642529296875,
|
||
|
|
"completions/min_length": 90.6,
|
||
|
|
"completions/min_terminated_length": 90.6,
|
||
|
|
"epoch": 0.976,
|
||
|
|
"grad_norm": 0.0008364045643247664,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 1020933056.0,
|
||
|
|
"reward": 1.1579480171203613,
|
||
|
|
"reward_std": 0.08691856116056443,
|
||
|
|
"rewards/accuracy_reward": 0.50361328125,
|
||
|
|
"rewards/brier_reward": 0.8128007888793946,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4667675793170929,
|
||
|
|
"rewards/smce": -0.04212206564843655,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 612.8,
|
||
|
|
"completions/max_terminated_length": 612.8,
|
||
|
|
"completions/mean_length": 171.72080078125,
|
||
|
|
"completions/mean_terminated_length": 171.72080078125,
|
||
|
|
"completions/min_length": 86.8,
|
||
|
|
"completions/min_terminated_length": 86.8,
|
||
|
|
"epoch": 0.992,
|
||
|
|
"grad_norm": 0.0009284996194764972,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 1037819957.0,
|
||
|
|
"reward": 1.1416526317596436,
|
||
|
|
"reward_std": 0.10318480283021927,
|
||
|
|
"rewards/accuracy_reward": 0.4888671875,
|
||
|
|
"rewards/brier_reward": 0.7947907686233521,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4984033167362213,
|
||
|
|
"rewards/smce": -0.03536987099796533,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000244140625,
|
||
|
|
"completions/max_length": 956.0,
|
||
|
|
"completions/max_terminated_length": 389.0,
|
||
|
|
"completions/mean_length": 169.3539276123047,
|
||
|
|
"completions/mean_terminated_length": 169.02008056640625,
|
||
|
|
"completions/min_length": 84.5,
|
||
|
|
"completions/min_terminated_length": 84.5,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"num_tokens": 1044519143.0,
|
||
|
|
"reward": 1.129215657711029,
|
||
|
|
"reward_std": 0.10631396621465683,
|
||
|
|
"rewards/accuracy_reward": 0.47412109375,
|
||
|
|
"rewards/brier_reward": 0.7854089140892029,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999755859375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5580566227436066,
|
||
|
|
"rewards/smce": -0.08556640334427357,
|
||
|
|
"step": 312,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.003961190384311171,
|
||
|
|
"train_runtime": 42522.1813,
|
||
|
|
"train_samples_per_second": 0.47,
|
||
|
|
"train_steps_per_second": 0.007
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 312,
|
||
|
|
"num_input_tokens_seen": 1044519143,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 60,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|