1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9984,
|
|
"eval_steps": 50,
|
|
"global_step": 312,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11220703125,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1528.6,
|
|
"completions/mean_length": 466.6302734375,
|
|
"completions/mean_terminated_length": 331.60613403320315,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"epoch": 0.016,
|
|
"grad_norm": 0.0026785461232066154,
|
|
"learning_rate": 3.1249999999999997e-07,
|
|
"loss": 0.0375,
|
|
"num_tokens": 19622326.0,
|
|
"reward": 0.09751158058643342,
|
|
"reward_std": 0.4065922200679779,
|
|
"rewards/accuracy_reward": 0.06201171875,
|
|
"rewards/brier_reward": 0.10872447341680527,
|
|
"rewards/confidence_one_or_zero": 0.41396484375,
|
|
"rewards/format_reward": 0.259375,
|
|
"rewards/mean_confidence_reward": 0.532010293006897,
|
|
"rewards/smce": -0.47017982006073,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10126953125,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1532.6,
|
|
"completions/mean_length": 446.04921875,
|
|
"completions/mean_terminated_length": 323.296875,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"epoch": 0.032,
|
|
"grad_norm": 0.00424788985401392,
|
|
"learning_rate": 6.249999999999999e-07,
|
|
"loss": 0.0371,
|
|
"num_tokens": 39290222.0,
|
|
"reward": 0.11004470586776734,
|
|
"reward_std": 0.4158190131187439,
|
|
"rewards/accuracy_reward": 0.06279296875,
|
|
"rewards/brier_reward": 0.11651045382022858,
|
|
"rewards/confidence_one_or_zero": 0.43310546875,
|
|
"rewards/format_reward": 0.28916015625,
|
|
"rewards/mean_confidence_reward": 0.559346604347229,
|
|
"rewards/smce": -0.49675229787826536,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05732421875,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1501.2,
|
|
"completions/mean_length": 317.6384765625,
|
|
"completions/mean_terminated_length": 243.9247039794922,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"epoch": 0.048,
|
|
"grad_norm": 0.004304820206016302,
|
|
"learning_rate": 9.374999999999999e-07,
|
|
"loss": 0.0425,
|
|
"num_tokens": 57591576.0,
|
|
"reward": 0.2621927112340927,
|
|
"reward_std": 0.4705701291561127,
|
|
"rewards/accuracy_reward": 0.10595703125,
|
|
"rewards/brier_reward": 0.19967454373836518,
|
|
"rewards/confidence_one_or_zero": 0.5154296875,
|
|
"rewards/format_reward": 0.50380859375,
|
|
"rewards/mean_confidence_reward": 0.6753389239311218,
|
|
"rewards/smce": -0.5701142072677612,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.02685546875,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1411.4,
|
|
"completions/mean_length": 222.1150390625,
|
|
"completions/mean_terminated_length": 186.05282897949218,
|
|
"completions/min_length": 8.8,
|
|
"completions/min_terminated_length": 8.8,
|
|
"epoch": 0.064,
|
|
"grad_norm": 0.003465007059276104,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0317,
|
|
"num_tokens": 74784434.0,
|
|
"reward": 0.5405649662017822,
|
|
"reward_std": 0.463888019323349,
|
|
"rewards/accuracy_reward": 0.1904296875,
|
|
"rewards/brier_reward": 0.35292364954948424,
|
|
"rewards/confidence_one_or_zero": 0.53916015625,
|
|
"rewards/format_reward": 0.8271484375,
|
|
"rewards/mean_confidence_reward": 0.7663158893585205,
|
|
"rewards/smce": -0.5787488460540772,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.012890625,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1265.8,
|
|
"completions/mean_length": 169.68251953125,
|
|
"completions/mean_terminated_length": 151.84535522460936,
|
|
"completions/min_length": 29.4,
|
|
"completions/min_terminated_length": 29.4,
|
|
"epoch": 0.08,
|
|
"grad_norm": 0.003266093088313937,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0177,
|
|
"num_tokens": 91455135.0,
|
|
"reward": 0.6993292093276977,
|
|
"reward_std": 0.40993152260780336,
|
|
"rewards/accuracy_reward": 0.2166015625,
|
|
"rewards/brier_reward": 0.4692054033279419,
|
|
"rewards/confidence_one_or_zero": 0.443359375,
|
|
"rewards/format_reward": 0.96181640625,
|
|
"rewards/mean_confidence_reward": 0.7073621034622193,
|
|
"rewards/smce": -0.49793441891670226,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0080078125,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1338.0,
|
|
"completions/mean_length": 152.65791015625,
|
|
"completions/mean_terminated_length": 141.49590148925782,
|
|
"completions/min_length": 34.2,
|
|
"completions/min_terminated_length": 34.2,
|
|
"epoch": 0.096,
|
|
"grad_norm": 0.00212898850440979,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0139,
|
|
"num_tokens": 108062960.0,
|
|
"reward": 0.8696157813072205,
|
|
"reward_std": 0.3268528938293457,
|
|
"rewards/accuracy_reward": 0.23359375,
|
|
"rewards/brier_reward": 0.6494227528572083,
|
|
"rewards/confidence_one_or_zero": 0.310546875,
|
|
"rewards/format_reward": 0.9818359375,
|
|
"rewards/mean_confidence_reward": 0.4570514678955078,
|
|
"rewards/smce": -0.25124485343694686,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.002734375,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 960.4,
|
|
"completions/mean_length": 131.7365234375,
|
|
"completions/mean_terminated_length": 127.88609161376954,
|
|
"completions/min_length": 32.6,
|
|
"completions/min_terminated_length": 32.6,
|
|
"epoch": 0.112,
|
|
"grad_norm": 0.0015331514878198504,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0054,
|
|
"num_tokens": 124521414.0,
|
|
"reward": 0.9907561421394349,
|
|
"reward_std": 0.1528225064277649,
|
|
"rewards/accuracy_reward": 0.2646484375,
|
|
"rewards/brier_reward": 0.7692283868789673,
|
|
"rewards/confidence_one_or_zero": 0.3392578125,
|
|
"rewards/format_reward": 0.9931640625,
|
|
"rewards/mean_confidence_reward": 0.21056363880634307,
|
|
"rewards/smce": -0.09105930775403977,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00146484375,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 664.8,
|
|
"completions/mean_length": 125.91826171875,
|
|
"completions/mean_terminated_length": 123.85120544433593,
|
|
"completions/min_length": 40.6,
|
|
"completions/min_terminated_length": 40.6,
|
|
"epoch": 0.128,
|
|
"grad_norm": 0.000771750696003437,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 140727489.0,
|
|
"reward": 0.9993586897850036,
|
|
"reward_std": 0.09848072528839111,
|
|
"rewards/accuracy_reward": 0.2533203125,
|
|
"rewards/brier_reward": 0.791638457775116,
|
|
"rewards/confidence_one_or_zero": 0.256640625,
|
|
"rewards/format_reward": 0.99677734375,
|
|
"rewards/mean_confidence_reward": 0.18074644207954407,
|
|
"rewards/smce": -0.08603915795683861,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0005859375,
|
|
"completions/max_length": 1335.4,
|
|
"completions/max_terminated_length": 510.2,
|
|
"completions/mean_length": 123.43896484375,
|
|
"completions/mean_terminated_length": 122.61112060546876,
|
|
"completions/min_length": 45.2,
|
|
"completions/min_terminated_length": 45.2,
|
|
"epoch": 0.144,
|
|
"grad_norm": 0.0008707439992576838,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 156941936.0,
|
|
"reward": 1.0172405004501344,
|
|
"reward_std": 0.09546828716993332,
|
|
"rewards/accuracy_reward": 0.2875,
|
|
"rewards/brier_reward": 0.7847882270812988,
|
|
"rewards/confidence_one_or_zero": 0.12060546875,
|
|
"rewards/format_reward": 0.99716796875,
|
|
"rewards/mean_confidence_reward": 0.2260705977678299,
|
|
"rewards/smce": -0.06995189264416694,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00078125,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 543.6,
|
|
"completions/mean_length": 127.2958984375,
|
|
"completions/mean_terminated_length": 126.1939910888672,
|
|
"completions/min_length": 55.4,
|
|
"completions/min_terminated_length": 55.4,
|
|
"epoch": 0.16,
|
|
"grad_norm": 0.0015283646062016487,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0019,
|
|
"num_tokens": 173266374.0,
|
|
"reward": 1.0233920097351075,
|
|
"reward_std": 0.15360372960567475,
|
|
"rewards/accuracy_reward": 0.2744140625,
|
|
"rewards/brier_reward": 0.7956377387046814,
|
|
"rewards/confidence_one_or_zero": 0.04130859375,
|
|
"rewards/format_reward": 0.9982421875,
|
|
"rewards/mean_confidence_reward": 0.2933915197849274,
|
|
"rewards/smce": -0.043021409586071965,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.16,
|
|
"eval_completions/clipped_ratio": 0.002049180327868827,
|
|
"eval_completions/max_length": 909.5,
|
|
"eval_completions/max_terminated_length": 406.0,
|
|
"eval_completions/mean_length": 132.36570739746094,
|
|
"eval_completions/mean_terminated_length": 129.48561096191406,
|
|
"eval_completions/min_length": 62.5,
|
|
"eval_completions/min_terminated_length": 62.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 173266374.0,
|
|
"eval_reward": 0.9988523125648499,
|
|
"eval_reward_std": 0.2654736191034317,
|
|
"eval_rewards/accuracy_reward": 0.232421875,
|
|
"eval_rewards/brier_reward": 0.8115327954292297,
|
|
"eval_rewards/confidence_one_or_zero": 0.01953125,
|
|
"eval_rewards/format_reward": 0.998046875,
|
|
"eval_rewards/mean_confidence_reward": 0.3196093738079071,
|
|
"eval_rewards/smce": -0.08859512209892273,
|
|
"eval_runtime": 21.2363,
|
|
"eval_samples_per_second": 23.545,
|
|
"eval_steps_per_second": 0.094,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 676.2,
|
|
"completions/max_terminated_length": 482.0,
|
|
"completions/mean_length": 126.12138671875,
|
|
"completions/mean_terminated_length": 125.98388366699218,
|
|
"completions/min_length": 53.6,
|
|
"completions/min_terminated_length": 53.6,
|
|
"epoch": 0.176,
|
|
"grad_norm": 0.001403786358423531,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 189794977.0,
|
|
"reward": 1.0303616762161254,
|
|
"reward_std": 0.1711978167295456,
|
|
"rewards/accuracy_reward": 0.28095703125,
|
|
"rewards/brier_reward": 0.798363721370697,
|
|
"rewards/confidence_one_or_zero": 0.02021484375,
|
|
"rewards/format_reward": 0.9990234375,
|
|
"rewards/mean_confidence_reward": 0.3105536162853241,
|
|
"rewards/smce": -0.03524293936789036,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 938.8,
|
|
"completions/max_terminated_length": 484.0,
|
|
"completions/mean_length": 126.25869140625,
|
|
"completions/mean_terminated_length": 125.98303680419922,
|
|
"completions/min_length": 55.6,
|
|
"completions/min_terminated_length": 55.6,
|
|
"epoch": 0.192,
|
|
"grad_norm": 0.0014057139633223414,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 205902682.0,
|
|
"reward": 1.0450642347335815,
|
|
"reward_std": 0.0979317307472229,
|
|
"rewards/accuracy_reward": 0.33349609375,
|
|
"rewards/brier_reward": 0.7794028878211975,
|
|
"rewards/confidence_one_or_zero": 0.00712890625,
|
|
"rewards/format_reward": 0.99912109375,
|
|
"rewards/mean_confidence_reward": 0.3114392042160034,
|
|
"rewards/smce": -0.04378440342843533,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.000390625,
|
|
"completions/max_length": 1121.8,
|
|
"completions/max_terminated_length": 461.8,
|
|
"completions/mean_length": 129.217578125,
|
|
"completions/mean_terminated_length": 128.668408203125,
|
|
"completions/min_length": 56.4,
|
|
"completions/min_terminated_length": 56.4,
|
|
"epoch": 0.208,
|
|
"grad_norm": 0.0007134112529456615,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 222258094.0,
|
|
"reward": 1.0534921646118165,
|
|
"reward_std": 0.09793903082609176,
|
|
"rewards/accuracy_reward": 0.35361328125,
|
|
"rewards/brier_reward": 0.7721356630325318,
|
|
"rewards/confidence_one_or_zero": 0.0029296875,
|
|
"rewards/format_reward": 0.9990234375,
|
|
"rewards/mean_confidence_reward": 0.32707850337028505,
|
|
"rewards/smce": -0.03557743579149246,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 806.2,
|
|
"completions/max_terminated_length": 561.2,
|
|
"completions/mean_length": 126.786328125,
|
|
"completions/mean_terminated_length": 126.64864959716797,
|
|
"completions/min_length": 56.8,
|
|
"completions/min_terminated_length": 56.8,
|
|
"epoch": 0.224,
|
|
"grad_norm": 0.0014312885468825698,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 238709570.0,
|
|
"reward": 1.0552095890045166,
|
|
"reward_std": 0.11882918179035187,
|
|
"rewards/accuracy_reward": 0.34658203125,
|
|
"rewards/brier_reward": 0.7787379264831543,
|
|
"rewards/confidence_one_or_zero": 0.0013671875,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.3532362937927246,
|
|
"rewards/smce": -0.029021762683987616,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 750.0,
|
|
"completions/max_terminated_length": 540.4,
|
|
"completions/mean_length": 128.212109375,
|
|
"completions/mean_terminated_length": 128.07445831298827,
|
|
"completions/min_length": 56.0,
|
|
"completions/min_terminated_length": 56.0,
|
|
"epoch": 0.24,
|
|
"grad_norm": 0.0015375501243397593,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 255274142.0,
|
|
"reward": 1.0612101793289184,
|
|
"reward_std": 0.14273426681756973,
|
|
"rewards/accuracy_reward": 0.39052734375,
|
|
"rewards/brier_reward": 0.7600910544395447,
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
|
"rewards/format_reward": 0.99931640625,
|
|
"rewards/mean_confidence_reward": 0.3782422184944153,
|
|
"rewards/smce": -0.055030280724167824,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 389.4,
|
|
"completions/max_terminated_length": 389.4,
|
|
"completions/mean_length": 127.03037109375,
|
|
"completions/mean_terminated_length": 127.03037109375,
|
|
"completions/min_length": 57.2,
|
|
"completions/min_terminated_length": 57.2,
|
|
"epoch": 0.256,
|
|
"grad_norm": 0.0014671719400212169,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 271629749.0,
|
|
"reward": 1.0515930891036986,
|
|
"reward_std": 0.1513766437768936,
|
|
"rewards/accuracy_reward": 0.35771484375,
|
|
"rewards/brier_reward": 0.7751455187797547,
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.3833678364753723,
|
|
"rewards/smce": -0.059154579788446425,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 609.4,
|
|
"completions/max_terminated_length": 382.0,
|
|
"completions/mean_length": 133.298828125,
|
|
"completions/mean_terminated_length": 133.16205444335938,
|
|
"completions/min_length": 60.6,
|
|
"completions/min_terminated_length": 60.6,
|
|
"epoch": 0.272,
|
|
"grad_norm": 0.0006801378331147134,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 287960425.0,
|
|
"reward": 1.0686883926391602,
|
|
"reward_std": 0.08673371747136116,
|
|
"rewards/accuracy_reward": 0.3943359375,
|
|
"rewards/brier_reward": 0.7641499638557434,
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
|
"rewards/format_reward": 0.99951171875,
|
|
"rewards/mean_confidence_reward": 0.36116114258766174,
|
|
"rewards/smce": -0.041243164986372,
|
|
"step": 85
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 601.4,
|
|
"completions/max_terminated_length": 365.2,
|
|
"completions/mean_length": 131.62919921875,
|
|
"completions/mean_terminated_length": 131.4923095703125,
|
|
"completions/min_length": 58.8,
|
|
"completions/min_terminated_length": 58.8,
|
|
"epoch": 0.288,
|
|
"grad_norm": 0.0007481302018277347,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 304266484.0,
|
|
"reward": 1.0655381202697753,
|
|
"reward_std": 0.10698467046022415,
|
|
"rewards/accuracy_reward": 0.38828125,
|
|
"rewards/brier_reward": 0.7688133239746093,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.3731468141078949,
|
|
"rewards/smce": -0.05125699490308762,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 864.2,
|
|
"completions/max_terminated_length": 418.8,
|
|
"completions/mean_length": 132.52294921875,
|
|
"completions/mean_terminated_length": 132.2486785888672,
|
|
"completions/min_length": 58.2,
|
|
"completions/min_terminated_length": 58.2,
|
|
"epoch": 0.304,
|
|
"grad_norm": 0.0016072518192231655,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 320553471.0,
|
|
"reward": 1.072618842124939,
|
|
"reward_std": 0.0957090824842453,
|
|
"rewards/accuracy_reward": 0.39638671875,
|
|
"rewards/brier_reward": 0.7660355925559997,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.39573111534118655,
|
|
"rewards/smce": -0.03378484845161438,
|
|
"step": 95
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 726.0,
|
|
"completions/max_terminated_length": 483.8,
|
|
"completions/mean_length": 134.47216796875,
|
|
"completions/mean_terminated_length": 134.33522033691406,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"epoch": 0.32,
|
|
"grad_norm": 0.0012717916397377849,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 337019170.0,
|
|
"reward": 1.0765431642532348,
|
|
"reward_std": 0.12678306698799133,
|
|
"rewards/accuracy_reward": 0.414453125,
|
|
"rewards/brier_reward": 0.768175745010376,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.42739094495773317,
|
|
"rewards/smce": -0.058696174249053004,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.32,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 261.0,
|
|
"eval_completions/max_terminated_length": 261.0,
|
|
"eval_completions/mean_length": 137.20244598388672,
|
|
"eval_completions/mean_terminated_length": 137.20244598388672,
|
|
"eval_completions/min_length": 68.5,
|
|
"eval_completions/min_terminated_length": 68.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 337019170.0,
|
|
"eval_reward": 1.0082088112831116,
|
|
"eval_reward_std": 0.3224063217639923,
|
|
"eval_rewards/accuracy_reward": 0.3046875,
|
|
"eval_rewards/brier_reward": 0.7768570184707642,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.43464845418930054,
|
|
"eval_rewards/smce": -0.13025547564029694,
|
|
"eval_runtime": 10.2855,
|
|
"eval_samples_per_second": 48.612,
|
|
"eval_steps_per_second": 0.194,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 871.0,
|
|
"completions/max_terminated_length": 389.2,
|
|
"completions/mean_length": 133.99482421875,
|
|
"completions/mean_terminated_length": 133.72105102539064,
|
|
"completions/min_length": 55.6,
|
|
"completions/min_terminated_length": 55.6,
|
|
"epoch": 0.336,
|
|
"grad_norm": 0.0007206884911283851,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 353113709.0,
|
|
"reward": 1.0716354370117187,
|
|
"reward_std": 0.13324915170669555,
|
|
"rewards/accuracy_reward": 0.40849609375,
|
|
"rewards/brier_reward": 0.7654546618461608,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.4270427763462067,
|
|
"rewards/smce": -0.06077566109597683,
|
|
"step": 105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 681.4,
|
|
"completions/max_terminated_length": 465.2,
|
|
"completions/mean_length": 137.42919921875,
|
|
"completions/mean_terminated_length": 137.29295349121094,
|
|
"completions/min_length": 60.8,
|
|
"completions/min_terminated_length": 60.8,
|
|
"epoch": 0.352,
|
|
"grad_norm": 0.0013820958556607366,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 369781400.0,
|
|
"reward": 1.0609832763671876,
|
|
"reward_std": 0.14019967019557952,
|
|
"rewards/accuracy_reward": 0.36474609375,
|
|
"rewards/brier_reward": 0.7737235069274903,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.39401292204856875,
|
|
"rewards/smce": -0.032421684823930264,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 665.6,
|
|
"completions/max_terminated_length": 442.0,
|
|
"completions/mean_length": 142.323046875,
|
|
"completions/mean_terminated_length": 142.18718872070312,
|
|
"completions/min_length": 61.6,
|
|
"completions/min_terminated_length": 61.6,
|
|
"epoch": 0.368,
|
|
"grad_norm": 0.0005504830041900277,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 386304260.0,
|
|
"reward": 1.0715895414352417,
|
|
"reward_std": 0.09788179844617843,
|
|
"rewards/accuracy_reward": 0.38828125,
|
|
"rewards/brier_reward": 0.7760221242904664,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.383696848154068,
|
|
"rewards/smce": -0.04146903231739998,
|
|
"step": 115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 981.8,
|
|
"completions/max_terminated_length": 523.0,
|
|
"completions/mean_length": 149.26435546875,
|
|
"completions/mean_terminated_length": 148.9933074951172,
|
|
"completions/min_length": 61.8,
|
|
"completions/min_terminated_length": 61.8,
|
|
"epoch": 0.384,
|
|
"grad_norm": 0.0007317579002119601,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 402689239.0,
|
|
"reward": 1.082389736175537,
|
|
"reward_std": 0.08156476095318795,
|
|
"rewards/accuracy_reward": 0.41943359375,
|
|
"rewards/brier_reward": 0.7709457755088807,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.379436719417572,
|
|
"rewards/smce": -0.05081076994538307,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 418.0,
|
|
"completions/max_terminated_length": 418.0,
|
|
"completions/mean_length": 154.80625,
|
|
"completions/mean_terminated_length": 154.80625,
|
|
"completions/min_length": 64.4,
|
|
"completions/min_terminated_length": 64.4,
|
|
"epoch": 0.4,
|
|
"grad_norm": 0.001232767477631569,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 419310903.0,
|
|
"reward": 1.0757294654846192,
|
|
"reward_std": 0.12325254082679749,
|
|
"rewards/accuracy_reward": 0.39443359375,
|
|
"rewards/brier_reward": 0.7765468120574951,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4125796616077423,
|
|
"rewards/smce": -0.0388492189347744,
|
|
"step": 125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00068359375,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 475.6,
|
|
"completions/mean_length": 161.99873046875,
|
|
"completions/mean_terminated_length": 161.05875549316406,
|
|
"completions/min_length": 59.6,
|
|
"completions/min_terminated_length": 59.6,
|
|
"epoch": 0.416,
|
|
"grad_norm": 0.0006661260849796236,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 435850954.0,
|
|
"reward": 1.082474160194397,
|
|
"reward_std": 0.1305944874882698,
|
|
"rewards/accuracy_reward": 0.40322265625,
|
|
"rewards/brier_reward": 0.7749996423721314,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9990234375,
|
|
"rewards/mean_confidence_reward": 0.41062086820602417,
|
|
"rewards/smce": -0.02459658682346344,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 668.6,
|
|
"completions/max_terminated_length": 445.4,
|
|
"completions/mean_length": 166.15087890625,
|
|
"completions/mean_terminated_length": 166.01694946289064,
|
|
"completions/min_length": 70.0,
|
|
"completions/min_terminated_length": 70.0,
|
|
"epoch": 0.432,
|
|
"grad_norm": 0.0005438687512651086,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 452566675.0,
|
|
"reward": 1.0882618188858033,
|
|
"reward_std": 0.09735962152481079,
|
|
"rewards/accuracy_reward": 0.4267578125,
|
|
"rewards/brier_reward": 0.7717886209487915,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.41006878614425657,
|
|
"rewards/smce": -0.043851888924837115,
|
|
"step": 135
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 956.2,
|
|
"completions/max_terminated_length": 521.0,
|
|
"completions/mean_length": 171.17646484375,
|
|
"completions/mean_terminated_length": 170.77630920410155,
|
|
"completions/min_length": 73.4,
|
|
"completions/min_terminated_length": 73.4,
|
|
"epoch": 0.448,
|
|
"grad_norm": 0.0005960149574093521,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 469272322.0,
|
|
"reward": 1.085831379890442,
|
|
"reward_std": 0.12048186510801315,
|
|
"rewards/accuracy_reward": 0.41865234375,
|
|
"rewards/brier_reward": 0.7699586391448975,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.41828733682632446,
|
|
"rewards/smce": -0.03331212252378464,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 896.6,
|
|
"completions/max_terminated_length": 487.2,
|
|
"completions/mean_length": 175.76884765625,
|
|
"completions/mean_terminated_length": 175.37088317871093,
|
|
"completions/min_length": 73.8,
|
|
"completions/min_terminated_length": 73.8,
|
|
"epoch": 0.464,
|
|
"grad_norm": 0.001222067279741168,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0015,
|
|
"num_tokens": 486243011.0,
|
|
"reward": 1.057556962966919,
|
|
"reward_std": 0.1313529819250107,
|
|
"rewards/accuracy_reward": 0.36416015625,
|
|
"rewards/brier_reward": 0.7829479813575745,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.411750328540802,
|
|
"rewards/smce": -0.06340402290225029,
|
|
"step": 145
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 921.0,
|
|
"completions/max_terminated_length": 477.6,
|
|
"completions/mean_length": 175.8951171875,
|
|
"completions/mean_terminated_length": 175.4965362548828,
|
|
"completions/min_length": 76.4,
|
|
"completions/min_terminated_length": 76.4,
|
|
"epoch": 0.48,
|
|
"grad_norm": 0.0006074010743759573,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 503092209.0,
|
|
"reward": 1.083800721168518,
|
|
"reward_std": 0.09710027799010276,
|
|
"rewards/accuracy_reward": 0.42744140625,
|
|
"rewards/brier_reward": 0.7678929805755615,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.3819667637348175,
|
|
"rewards/smce": -0.05488146059215069,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.48,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 346.0,
|
|
"eval_completions/max_terminated_length": 346.0,
|
|
"eval_completions/mean_length": 179.1521453857422,
|
|
"eval_completions/mean_terminated_length": 179.1521453857422,
|
|
"eval_completions/min_length": 83.5,
|
|
"eval_completions/min_terminated_length": 83.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 503092209.0,
|
|
"eval_reward": 1.055518388748169,
|
|
"eval_reward_std": 0.29327547550201416,
|
|
"eval_rewards/accuracy_reward": 0.333984375,
|
|
"eval_rewards/brier_reward": 0.793656051158905,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.3653808832168579,
|
|
"eval_rewards/smce": -0.03320850059390068,
|
|
"eval_runtime": 11.8844,
|
|
"eval_samples_per_second": 42.072,
|
|
"eval_steps_per_second": 0.168,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 498.4,
|
|
"completions/max_terminated_length": 498.4,
|
|
"completions/mean_length": 175.06181640625,
|
|
"completions/mean_terminated_length": 175.06181640625,
|
|
"completions/min_length": 79.0,
|
|
"completions/min_terminated_length": 79.0,
|
|
"epoch": 0.496,
|
|
"grad_norm": 0.0006104473723098636,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 520192682.0,
|
|
"reward": 1.0852439403533936,
|
|
"reward_std": 0.057844220846891406,
|
|
"rewards/accuracy_reward": 0.4291015625,
|
|
"rewards/brier_reward": 0.7675500392913819,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.3779485821723938,
|
|
"rewards/smce": -0.05232905186712742,
|
|
"step": 155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 437.6,
|
|
"completions/max_terminated_length": 437.6,
|
|
"completions/mean_length": 174.67958984375,
|
|
"completions/mean_terminated_length": 174.67958984375,
|
|
"completions/min_length": 82.6,
|
|
"completions/min_terminated_length": 82.6,
|
|
"epoch": 0.512,
|
|
"grad_norm": 0.0012512748362496495,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 537127065.0,
|
|
"reward": 1.099705195426941,
|
|
"reward_std": 0.1057943195104599,
|
|
"rewards/accuracy_reward": 0.44609375,
|
|
"rewards/brier_reward": 0.7742470264434814,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.42951337695121766,
|
|
"rewards/smce": -0.04186231698840857,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 964.8,
|
|
"completions/max_terminated_length": 511.0,
|
|
"completions/mean_length": 176.96630859375,
|
|
"completions/mean_terminated_length": 176.70090942382814,
|
|
"completions/min_length": 81.6,
|
|
"completions/min_terminated_length": 81.6,
|
|
"epoch": 0.528,
|
|
"grad_norm": 0.0014094150392338634,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 553968736.0,
|
|
"reward": 1.089972233772278,
|
|
"reward_std": 0.16099656820297242,
|
|
"rewards/accuracy_reward": 0.4208984375,
|
|
"rewards/brier_reward": 0.7824561715126037,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.46478373408317564,
|
|
"rewards/smce": -0.04643128663301468,
|
|
"step": 165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 880.4,
|
|
"completions/max_terminated_length": 468.6,
|
|
"completions/mean_length": 174.324609375,
|
|
"completions/mean_terminated_length": 173.92534790039062,
|
|
"completions/min_length": 81.6,
|
|
"completions/min_terminated_length": 81.6,
|
|
"epoch": 0.544,
|
|
"grad_norm": 0.0006505737546831369,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 570917404.0,
|
|
"reward": 1.1095312356948852,
|
|
"reward_std": 0.11838259994983673,
|
|
"rewards/accuracy_reward": 0.46826171875,
|
|
"rewards/brier_reward": 0.7663807153701783,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.4589267909526825,
|
|
"rewards/smce": -0.030575690418481828,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 897.6,
|
|
"completions/max_terminated_length": 467.6,
|
|
"completions/mean_length": 172.4845703125,
|
|
"completions/mean_terminated_length": 172.0850830078125,
|
|
"completions/min_length": 77.4,
|
|
"completions/min_terminated_length": 77.4,
|
|
"epoch": 0.56,
|
|
"grad_norm": 0.0012473545502871275,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 587505054.0,
|
|
"reward": 1.086290669441223,
|
|
"reward_std": 0.14059744775295258,
|
|
"rewards/accuracy_reward": 0.41142578125,
|
|
"rewards/brier_reward": 0.7802677392959595,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.44028844833374026,
|
|
"rewards/smce": -0.03764005638659,
|
|
"step": 175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.000390625,
|
|
"completions/max_length": 1094.4,
|
|
"completions/max_terminated_length": 501.2,
|
|
"completions/mean_length": 173.42021484375,
|
|
"completions/mean_terminated_length": 172.88742065429688,
|
|
"completions/min_length": 80.0,
|
|
"completions/min_terminated_length": 80.0,
|
|
"epoch": 0.576,
|
|
"grad_norm": 0.0005928562022745609,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 604467501.0,
|
|
"reward": 1.0810141086578369,
|
|
"reward_std": 0.08423209637403488,
|
|
"rewards/accuracy_reward": 0.4126953125,
|
|
"rewards/brier_reward": 0.777400267124176,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.3991813659667969,
|
|
"rewards/smce": -0.05535511486232281,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 863.8,
|
|
"completions/max_terminated_length": 490.0,
|
|
"completions/mean_length": 172.092578125,
|
|
"completions/mean_terminated_length": 171.82615966796874,
|
|
"completions/min_length": 79.4,
|
|
"completions/min_terminated_length": 79.4,
|
|
"epoch": 0.592,
|
|
"grad_norm": 0.0005724472575820982,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 621397441.0,
|
|
"reward": 1.0966659307479858,
|
|
"reward_std": 0.05616953819990158,
|
|
"rewards/accuracy_reward": 0.43623046875,
|
|
"rewards/brier_reward": 0.7744009375572205,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.40447003245353697,
|
|
"rewards/smce": -0.03421012945473194,
|
|
"step": 185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 1109.8,
|
|
"completions/max_terminated_length": 437.0,
|
|
"completions/mean_length": 174.52080078125,
|
|
"completions/mean_terminated_length": 174.12187805175782,
|
|
"completions/min_length": 78.2,
|
|
"completions/min_terminated_length": 78.2,
|
|
"epoch": 0.608,
|
|
"grad_norm": 0.0029141635168343782,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 638184022.0,
|
|
"reward": 1.0967584371566772,
|
|
"reward_std": 0.11957666501402855,
|
|
"rewards/accuracy_reward": 0.42626953125,
|
|
"rewards/brier_reward": 0.7853588700294495,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.4422805607318878,
|
|
"rewards/smce": -0.03563878741115332,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 694.8,
|
|
"completions/max_terminated_length": 488.0,
|
|
"completions/mean_length": 178.27265625,
|
|
"completions/mean_terminated_length": 178.1399719238281,
|
|
"completions/min_length": 83.6,
|
|
"completions/min_terminated_length": 83.6,
|
|
"epoch": 0.624,
|
|
"grad_norm": 0.0013668371830135584,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 655353438.0,
|
|
"reward": 1.0849785327911377,
|
|
"reward_std": 0.14437882453203202,
|
|
"rewards/accuracy_reward": 0.4138671875,
|
|
"rewards/brier_reward": 0.7801495552062988,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.45922787189483644,
|
|
"rewards/smce": -0.047925911750644444,
|
|
"step": 195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 1099.6,
|
|
"completions/max_terminated_length": 477.2,
|
|
"completions/mean_length": 176.486328125,
|
|
"completions/mean_terminated_length": 176.0878173828125,
|
|
"completions/min_length": 84.8,
|
|
"completions/min_terminated_length": 84.8,
|
|
"epoch": 0.64,
|
|
"grad_norm": 0.0005942194839008152,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 672503346.0,
|
|
"reward": 1.117413091659546,
|
|
"reward_std": 0.09016603529453278,
|
|
"rewards/accuracy_reward": 0.48681640625,
|
|
"rewards/brier_reward": 0.7776980638504029,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.463961261510849,
|
|
"rewards/smce": -0.05859697572886944,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.64,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 385.5,
|
|
"eval_completions/max_terminated_length": 385.5,
|
|
"eval_completions/mean_length": 179.0931396484375,
|
|
"eval_completions/mean_terminated_length": 179.0931396484375,
|
|
"eval_completions/min_length": 93.0,
|
|
"eval_completions/min_terminated_length": 93.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 672503346.0,
|
|
"eval_reward": 1.0669192671775818,
|
|
"eval_reward_std": 0.3463890254497528,
|
|
"eval_rewards/accuracy_reward": 0.38671875,
|
|
"eval_rewards/brier_reward": 0.7844113707542419,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.4601171910762787,
|
|
"eval_rewards/smce": -0.0745849683880806,
|
|
"eval_runtime": 12.3788,
|
|
"eval_samples_per_second": 40.392,
|
|
"eval_steps_per_second": 0.162,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 455.2,
|
|
"completions/max_terminated_length": 455.2,
|
|
"completions/mean_length": 172.68984375,
|
|
"completions/mean_terminated_length": 172.68984375,
|
|
"completions/min_length": 78.0,
|
|
"completions/min_terminated_length": 78.0,
|
|
"epoch": 0.656,
|
|
"grad_norm": 0.001317652058787644,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 689128234.0,
|
|
"reward": 1.0755244255065919,
|
|
"reward_std": 0.14942533075809478,
|
|
"rewards/accuracy_reward": 0.40205078125,
|
|
"rewards/brier_reward": 0.7797590136528015,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4621377050876617,
|
|
"rewards/smce": -0.06132837850600481,
|
|
"step": 205
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 650.0,
|
|
"completions/max_terminated_length": 423.2,
|
|
"completions/mean_length": 167.75009765625,
|
|
"completions/mean_terminated_length": 167.48269653320312,
|
|
"completions/min_length": 77.4,
|
|
"completions/min_terminated_length": 77.4,
|
|
"epoch": 0.672,
|
|
"grad_norm": 0.0010439743055030704,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 705759435.0,
|
|
"reward": 1.0993054628372192,
|
|
"reward_std": 0.07033557817339897,
|
|
"rewards/accuracy_reward": 0.43154296875,
|
|
"rewards/brier_reward": 0.7837408065795899,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.4168268620967865,
|
|
"rewards/smce": -0.03276139870285988,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 426.0,
|
|
"completions/max_terminated_length": 426.0,
|
|
"completions/mean_length": 166.26279296875,
|
|
"completions/mean_terminated_length": 166.26279296875,
|
|
"completions/min_length": 79.4,
|
|
"completions/min_terminated_length": 79.4,
|
|
"epoch": 0.688,
|
|
"grad_norm": 0.0007595060160383582,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 722415886.0,
|
|
"reward": 1.0981730222702026,
|
|
"reward_std": 0.09036669880151749,
|
|
"rewards/accuracy_reward": 0.43310546875,
|
|
"rewards/brier_reward": 0.7850585222244263,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4186152458190918,
|
|
"rewards/smce": -0.04344206675887108,
|
|
"step": 215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 610.0,
|
|
"completions/max_terminated_length": 379.4,
|
|
"completions/mean_length": 166.392578125,
|
|
"completions/mean_terminated_length": 166.25871887207032,
|
|
"completions/min_length": 76.8,
|
|
"completions/min_terminated_length": 76.8,
|
|
"epoch": 0.704,
|
|
"grad_norm": 0.0007093519088812172,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 738985890.0,
|
|
"reward": 1.111887812614441,
|
|
"reward_std": 0.07709287628531455,
|
|
"rewards/accuracy_reward": 0.45732421875,
|
|
"rewards/brier_reward": 0.7776798009872437,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4373369514942169,
|
|
"rewards/smce": -0.02226333422586322,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 622.0,
|
|
"completions/max_terminated_length": 393.8,
|
|
"completions/mean_length": 169.74150390625,
|
|
"completions/mean_terminated_length": 169.47537841796876,
|
|
"completions/min_length": 77.6,
|
|
"completions/min_terminated_length": 77.6,
|
|
"epoch": 0.72,
|
|
"grad_norm": 0.0014726045774295926,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 755733899.0,
|
|
"reward": 1.1221982479095458,
|
|
"reward_std": 0.11447475254535674,
|
|
"rewards/accuracy_reward": 0.4787109375,
|
|
"rewards/brier_reward": 0.7866963624954224,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4943776249885559,
|
|
"rewards/smce": -0.041632843762636186,
|
|
"step": 225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 874.8,
|
|
"completions/max_terminated_length": 444.8,
|
|
"completions/mean_length": 176.96787109375,
|
|
"completions/mean_terminated_length": 176.70293579101562,
|
|
"completions/min_length": 82.6,
|
|
"completions/min_terminated_length": 82.6,
|
|
"epoch": 0.736,
|
|
"grad_norm": 0.0012901662848889828,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 772485634.0,
|
|
"reward": 1.1121197938919067,
|
|
"reward_std": 0.14175231754779816,
|
|
"rewards/accuracy_reward": 0.47734375,
|
|
"rewards/brier_reward": 0.7755715608596802,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.5298642635345459,
|
|
"rewards/smce": -0.056962878257036206,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 444.0,
|
|
"completions/max_terminated_length": 444.0,
|
|
"completions/mean_length": 181.18896484375,
|
|
"completions/mean_terminated_length": 181.18896484375,
|
|
"completions/min_length": 88.4,
|
|
"completions/min_terminated_length": 88.4,
|
|
"epoch": 0.752,
|
|
"grad_norm": 0.0012587190140038729,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 789568209.0,
|
|
"reward": 1.1084522962570191,
|
|
"reward_std": 0.13682172298431397,
|
|
"rewards/accuracy_reward": 0.45869140625,
|
|
"rewards/brier_reward": 0.7826623439788818,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4847588062286377,
|
|
"rewards/smce": -0.04890004470944405,
|
|
"step": 235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 871.8,
|
|
"completions/max_terminated_length": 433.6,
|
|
"completions/mean_length": 184.03701171875,
|
|
"completions/mean_terminated_length": 183.77239074707032,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"epoch": 0.768,
|
|
"grad_norm": 0.0006104763597249985,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 806385452.0,
|
|
"reward": 1.1183516025543212,
|
|
"reward_std": 0.07741678655147552,
|
|
"rewards/accuracy_reward": 0.4650390625,
|
|
"rewards/brier_reward": 0.7864425659179688,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4476728618144989,
|
|
"rewards/smce": -0.02916794307529926,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 472.0,
|
|
"completions/max_terminated_length": 472.0,
|
|
"completions/mean_length": 185.95849609375,
|
|
"completions/mean_terminated_length": 185.95849609375,
|
|
"completions/min_length": 90.6,
|
|
"completions/min_terminated_length": 90.6,
|
|
"epoch": 0.784,
|
|
"grad_norm": 0.0012980833416804671,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 823464035.0,
|
|
"reward": 1.1076786994934082,
|
|
"reward_std": 0.08230673745274544,
|
|
"rewards/accuracy_reward": 0.46953125,
|
|
"rewards/brier_reward": 0.7831770896911621,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.45920702815055847,
|
|
"rewards/smce": -0.07450821846723557,
|
|
"step": 245
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 431.0,
|
|
"completions/max_terminated_length": 431.0,
|
|
"completions/mean_length": 190.05078125,
|
|
"completions/mean_terminated_length": 190.05078125,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"epoch": 0.8,
|
|
"grad_norm": 0.0012545861536636949,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 840420715.0,
|
|
"reward": 1.1329187154769897,
|
|
"reward_std": 0.13519156575202942,
|
|
"rewards/accuracy_reward": 0.49365234375,
|
|
"rewards/brier_reward": 0.7884702205657959,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.506415045261383,
|
|
"rewards/smce": -0.03257214445620775,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.8,
|
|
"eval_completions/clipped_ratio": 0.002049180327868827,
|
|
"eval_completions/max_length": 931.5,
|
|
"eval_completions/max_terminated_length": 334.5,
|
|
"eval_completions/mean_length": 196.33683013916016,
|
|
"eval_completions/mean_terminated_length": 193.58879852294922,
|
|
"eval_completions/min_length": 109.5,
|
|
"eval_completions/min_terminated_length": 109.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 840420715.0,
|
|
"eval_reward": 1.0556547045707703,
|
|
"eval_reward_std": 0.365094929933548,
|
|
"eval_rewards/accuracy_reward": 0.376953125,
|
|
"eval_rewards/brier_reward": 0.7935156226158142,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 0.998046875,
|
|
"eval_rewards/mean_confidence_reward": 0.4902343899011612,
|
|
"eval_rewards/smce": -0.11441406980156898,
|
|
"eval_runtime": 22.1298,
|
|
"eval_samples_per_second": 22.594,
|
|
"eval_steps_per_second": 0.09,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 460.4,
|
|
"completions/max_terminated_length": 460.4,
|
|
"completions/mean_length": 191.8078125,
|
|
"completions/mean_terminated_length": 191.8078125,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"epoch": 0.816,
|
|
"grad_norm": 0.0006870666402392089,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 857483995.0,
|
|
"reward": 1.1305972576141357,
|
|
"reward_std": 0.13415717780590058,
|
|
"rewards/accuracy_reward": 0.48408203125,
|
|
"rewards/brier_reward": 0.7903084397315979,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4990163028240204,
|
|
"rewards/smce": -0.0261984433978796,
|
|
"step": 255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 662.6,
|
|
"completions/max_terminated_length": 483.2,
|
|
"completions/mean_length": 201.5015625,
|
|
"completions/mean_terminated_length": 201.371142578125,
|
|
"completions/min_length": 96.8,
|
|
"completions/min_terminated_length": 96.8,
|
|
"epoch": 0.832,
|
|
"grad_norm": 0.0012498555006459355,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 874555723.0,
|
|
"reward": 1.1258383750915528,
|
|
"reward_std": 0.09739863574504852,
|
|
"rewards/accuracy_reward": 0.48115234375,
|
|
"rewards/brier_reward": 0.793649184703827,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.45032520294189454,
|
|
"rewards/smce": -0.04605602435767651,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 772.8,
|
|
"completions/max_terminated_length": 551.4,
|
|
"completions/mean_length": 204.1986328125,
|
|
"completions/mean_terminated_length": 204.06841125488282,
|
|
"completions/min_length": 100.6,
|
|
"completions/min_terminated_length": 100.6,
|
|
"epoch": 0.848,
|
|
"grad_norm": 0.0005774867604486644,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 891661085.0,
|
|
"reward": 1.120431137084961,
|
|
"reward_std": 0.09573130160570145,
|
|
"rewards/accuracy_reward": 0.4623046875,
|
|
"rewards/brier_reward": 0.7905663847923279,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.44633203744888306,
|
|
"rewards/smce": -0.023824118450284003,
|
|
"step": 265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 1015.2,
|
|
"completions/max_terminated_length": 626.0,
|
|
"completions/mean_length": 208.18095703125,
|
|
"completions/mean_terminated_length": 207.9215057373047,
|
|
"completions/min_length": 98.4,
|
|
"completions/min_terminated_length": 98.4,
|
|
"epoch": 0.864,
|
|
"grad_norm": 0.0007092739688232541,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 908779674.0,
|
|
"reward": 1.1397951364517211,
|
|
"reward_std": 0.08750861436128617,
|
|
"rewards/accuracy_reward": 0.50068359375,
|
|
"rewards/brier_reward": 0.7907498121261597,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4849375069141388,
|
|
"rewards/smce": -0.02329762177541852,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 940.4,
|
|
"completions/max_terminated_length": 498.2,
|
|
"completions/mean_length": 205.87783203125,
|
|
"completions/mean_terminated_length": 205.48865661621093,
|
|
"completions/min_length": 99.0,
|
|
"completions/min_terminated_length": 99.0,
|
|
"epoch": 0.88,
|
|
"grad_norm": 0.0006288170116022229,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 926034935.0,
|
|
"reward": 1.1090987920761108,
|
|
"reward_std": 0.11937552094459533,
|
|
"rewards/accuracy_reward": 0.44970703125,
|
|
"rewards/brier_reward": 0.7989086747169495,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.49709376096725466,
|
|
"rewards/smce": -0.06025214046239853,
|
|
"step": 275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 579.0,
|
|
"completions/max_terminated_length": 579.0,
|
|
"completions/mean_length": 208.03515625,
|
|
"completions/mean_terminated_length": 208.03515625,
|
|
"completions/min_length": 104.8,
|
|
"completions/min_terminated_length": 104.8,
|
|
"epoch": 0.896,
|
|
"grad_norm": 0.0010682296706363559,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 943276063.0,
|
|
"reward": 1.110878300666809,
|
|
"reward_std": 0.15439056754112243,
|
|
"rewards/accuracy_reward": 0.4568359375,
|
|
"rewards/brier_reward": 0.792157506942749,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.510614275932312,
|
|
"rewards/smce": -0.054475681856274605,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 822.8,
|
|
"completions/max_terminated_length": 630.6,
|
|
"completions/mean_length": 207.34013671875,
|
|
"completions/mean_terminated_length": 207.2098815917969,
|
|
"completions/min_length": 103.6,
|
|
"completions/min_terminated_length": 103.6,
|
|
"epoch": 0.912,
|
|
"grad_norm": 0.0005791044677607715,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 960450522.0,
|
|
"reward": 1.1124805688858033,
|
|
"reward_std": 0.10075956732034683,
|
|
"rewards/accuracy_reward": 0.4552734375,
|
|
"rewards/brier_reward": 0.7934512853622436,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4501240313053131,
|
|
"rewards/smce": -0.047138142958283426,
|
|
"step": 285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 740.2,
|
|
"completions/max_terminated_length": 519.0,
|
|
"completions/mean_length": 201.753515625,
|
|
"completions/mean_terminated_length": 201.6229217529297,
|
|
"completions/min_length": 102.4,
|
|
"completions/min_terminated_length": 102.4,
|
|
"epoch": 0.928,
|
|
"grad_norm": 0.0009222588851116598,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 977543294.0,
|
|
"reward": 1.121965217590332,
|
|
"reward_std": 0.08357778042554856,
|
|
"rewards/accuracy_reward": 0.48759765625,
|
|
"rewards/brier_reward": 0.7841106534004212,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.44317675828933717,
|
|
"rewards/smce": -0.055362095683813096,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.000390625,
|
|
"completions/max_length": 905.6,
|
|
"completions/max_terminated_length": 643.6,
|
|
"completions/mean_length": 201.8884765625,
|
|
"completions/mean_terminated_length": 201.3676971435547,
|
|
"completions/min_length": 91.2,
|
|
"completions/min_terminated_length": 91.2,
|
|
"epoch": 0.944,
|
|
"grad_norm": 0.0006336426013149321,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 994586056.0,
|
|
"reward": 1.1151194334030152,
|
|
"reward_std": 0.10739713907241821,
|
|
"rewards/accuracy_reward": 0.4759765625,
|
|
"rewards/brier_reward": 0.7890912055969238,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.4860449254512787,
|
|
"rewards/smce": -0.0688786506652832,
|
|
"step": 295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 841.2,
|
|
"completions/max_terminated_length": 659.8,
|
|
"completions/mean_length": 205.77861328125,
|
|
"completions/mean_terminated_length": 205.64851989746094,
|
|
"completions/min_length": 103.6,
|
|
"completions/min_terminated_length": 103.6,
|
|
"epoch": 0.96,
|
|
"grad_norm": 0.0012084617046639323,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0004,
|
|
"num_tokens": 1011633549.0,
|
|
"reward": 1.1156575441360475,
|
|
"reward_std": 0.13963824212551118,
|
|
"rewards/accuracy_reward": 0.4591796875,
|
|
"rewards/brier_reward": 0.7977803349494934,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.5077031373977661,
|
|
"rewards/smce": -0.051096287369728086,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.96,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 495.0,
|
|
"eval_completions/max_terminated_length": 495.0,
|
|
"eval_completions/mean_length": 209.3858184814453,
|
|
"eval_completions/mean_terminated_length": 209.3858184814453,
|
|
"eval_completions/min_length": 114.5,
|
|
"eval_completions/min_terminated_length": 114.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 1011633549.0,
|
|
"eval_reward": 1.0641183853149414,
|
|
"eval_reward_std": 0.3622732162475586,
|
|
"eval_rewards/accuracy_reward": 0.375,
|
|
"eval_rewards/brier_reward": 0.8058867156505585,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.4800390601158142,
|
|
"eval_rewards/smce": -0.10530157387256622,
|
|
"eval_runtime": 14.8357,
|
|
"eval_samples_per_second": 33.702,
|
|
"eval_steps_per_second": 0.135,
|
|
"step": 300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 912.4,
|
|
"completions/max_terminated_length": 546.8,
|
|
"completions/mean_length": 212.29677734375,
|
|
"completions/mean_terminated_length": 211.9076354980469,
|
|
"completions/min_length": 97.4,
|
|
"completions/min_terminated_length": 97.4,
|
|
"epoch": 0.976,
|
|
"grad_norm": 0.0005940425908192992,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 1028668588.0,
|
|
"reward": 1.1358384847640992,
|
|
"reward_std": 0.07964012026786804,
|
|
"rewards/accuracy_reward": 0.48720703125,
|
|
"rewards/brier_reward": 0.7967997074127198,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.47436524033546446,
|
|
"rewards/smce": -0.023880173824727534,
|
|
"step": 305
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 964.4,
|
|
"completions/max_terminated_length": 589.6,
|
|
"completions/mean_length": 216.59365234375,
|
|
"completions/mean_terminated_length": 216.3355255126953,
|
|
"completions/min_length": 104.6,
|
|
"completions/min_terminated_length": 104.6,
|
|
"epoch": 0.992,
|
|
"grad_norm": 0.0006660929648205638,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 1046014987.0,
|
|
"reward": 1.1179794788360595,
|
|
"reward_std": 0.1090321883559227,
|
|
"rewards/accuracy_reward": 0.4705078125,
|
|
"rewards/brier_reward": 0.7883034467697143,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4531269669532776,
|
|
"rewards/smce": -0.04531567171216011,
|
|
"step": 310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 567.0,
|
|
"completions/max_terminated_length": 567.0,
|
|
"completions/mean_length": 215.76505279541016,
|
|
"completions/mean_terminated_length": 215.76505279541016,
|
|
"completions/min_length": 104.5,
|
|
"completions/min_terminated_length": 104.5,
|
|
"epoch": 0.9984,
|
|
"num_tokens": 1052904757.0,
|
|
"reward": 1.1193369626998901,
|
|
"reward_std": 0.10750538855791092,
|
|
"rewards/accuracy_reward": 0.467529296875,
|
|
"rewards/brier_reward": 0.7913386523723602,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4925537258386612,
|
|
"rewards/smce": -0.040389697067439556,
|
|
"step": 312,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.003545210200872344,
|
|
"train_runtime": 43249.5131,
|
|
"train_samples_per_second": 0.462,
|
|
"train_steps_per_second": 0.007
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 312,
|
|
"num_input_tokens_seen": 1052904757,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 60,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|