1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"eval_steps": 50,
|
||
|
|
"global_step": 312,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.1095703125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1532.6,
|
||
|
|
"completions/mean_length": 468.423046875,
|
||
|
|
"completions/mean_terminated_length": 337.1365539550781,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.016,
|
||
|
|
"grad_norm": 0.0029375546146184206,
|
||
|
|
"learning_rate": 3.1249999999999997e-07,
|
||
|
|
"loss": 0.0267,
|
||
|
|
"num_tokens": 19640684.0,
|
||
|
|
"reward": -0.016939924284815788,
|
||
|
|
"reward_std": 0.4662073254585266,
|
||
|
|
"rewards/accuracy_reward": 0.0642578125,
|
||
|
|
"rewards/brier_reward": 0.11008564978837967,
|
||
|
|
"rewards/confidence_one_or_zero": 0.41435546875,
|
||
|
|
"rewards/format_reward": 0.2634765625,
|
||
|
|
"rewards/mean_confidence_reward": 0.5356804013252259,
|
||
|
|
"rewards/smce": -0.4717017710208893,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.10654296875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1530.8,
|
||
|
|
"completions/mean_length": 456.72236328125,
|
||
|
|
"completions/mean_terminated_length": 328.05751953125,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.032,
|
||
|
|
"grad_norm": 0.003135639475658536,
|
||
|
|
"learning_rate": 6.249999999999999e-07,
|
||
|
|
"loss": 0.0229,
|
||
|
|
"num_tokens": 39417873.0,
|
||
|
|
"reward": -0.0207392705604434,
|
||
|
|
"reward_std": 0.4668224513530731,
|
||
|
|
"rewards/accuracy_reward": 0.05888671875,
|
||
|
|
"rewards/brier_reward": 0.11468577980995179,
|
||
|
|
"rewards/confidence_one_or_zero": 0.428125,
|
||
|
|
"rewards/format_reward": 0.27978515625,
|
||
|
|
"rewards/mean_confidence_reward": 0.5534478187561035,
|
||
|
|
"rewards/smce": -0.49483814239501955,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.06904296875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1517.0,
|
||
|
|
"completions/mean_length": 344.80185546875,
|
||
|
|
"completions/mean_terminated_length": 257.0526123046875,
|
||
|
|
"completions/min_length": 1.2,
|
||
|
|
"completions/min_terminated_length": 1.2,
|
||
|
|
"epoch": 0.048,
|
||
|
|
"grad_norm": 0.0045189037919044495,
|
||
|
|
"learning_rate": 9.374999999999999e-07,
|
||
|
|
"loss": 0.0296,
|
||
|
|
"num_tokens": 57997380.0,
|
||
|
|
"reward": 0.10179619886912405,
|
||
|
|
"reward_std": 0.5125089168548584,
|
||
|
|
"rewards/accuracy_reward": 0.0970703125,
|
||
|
|
"rewards/brier_reward": 0.18347085118293763,
|
||
|
|
"rewards/confidence_one_or_zero": 0.48662109375,
|
||
|
|
"rewards/format_reward": 0.45732421875,
|
||
|
|
"rewards/mean_confidence_reward": 0.6305847048759461,
|
||
|
|
"rewards/smce": -0.534275221824646,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0271484375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1432.0,
|
||
|
|
"completions/mean_length": 220.9154296875,
|
||
|
|
"completions/mean_terminated_length": 184.3703186035156,
|
||
|
|
"completions/min_length": 2.6,
|
||
|
|
"completions/min_terminated_length": 2.6,
|
||
|
|
"epoch": 0.064,
|
||
|
|
"grad_norm": 0.00436444440856576,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.027,
|
||
|
|
"num_tokens": 75177954.0,
|
||
|
|
"reward": 0.40185180902481077,
|
||
|
|
"reward_std": 0.5494400143623352,
|
||
|
|
"rewards/accuracy_reward": 0.18125,
|
||
|
|
"rewards/brier_reward": 0.36000813245773317,
|
||
|
|
"rewards/confidence_one_or_zero": 0.5275390625,
|
||
|
|
"rewards/format_reward": 0.81787109375,
|
||
|
|
"rewards/mean_confidence_reward": 0.732852327823639,
|
||
|
|
"rewards/smce": -0.5554281115531922,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0171875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1267.0,
|
||
|
|
"completions/mean_length": 173.53408203125,
|
||
|
|
"completions/mean_terminated_length": 149.69837951660156,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"epoch": 0.08,
|
||
|
|
"grad_norm": 0.004786408040672541,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0181,
|
||
|
|
"num_tokens": 91888095.0,
|
||
|
|
"reward": 0.6115968227386475,
|
||
|
|
"reward_std": 0.5032546579837799,
|
||
|
|
"rewards/accuracy_reward": 0.209765625,
|
||
|
|
"rewards/brier_reward": 0.5036922097206116,
|
||
|
|
"rewards/confidence_one_or_zero": 0.43955078125,
|
||
|
|
"rewards/format_reward": 0.9505859375,
|
||
|
|
"rewards/mean_confidence_reward": 0.6410665035247802,
|
||
|
|
"rewards/smce": -0.4408522069454193,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00849609375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1187.0,
|
||
|
|
"completions/mean_length": 147.68828125,
|
||
|
|
"completions/mean_terminated_length": 135.78773193359376,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"epoch": 0.096,
|
||
|
|
"grad_norm": 0.001847111969254911,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0116,
|
||
|
|
"num_tokens": 108445031.0,
|
||
|
|
"reward": 0.8734211802482605,
|
||
|
|
"reward_std": 0.3274820476770401,
|
||
|
|
"rewards/accuracy_reward": 0.2228515625,
|
||
|
|
"rewards/brier_reward": 0.7075286149978638,
|
||
|
|
"rewards/confidence_one_or_zero": 0.415234375,
|
||
|
|
"rewards/format_reward": 0.9798828125,
|
||
|
|
"rewards/mean_confidence_reward": 0.31975880861282346,
|
||
|
|
"rewards/smce": -0.1634220853447914,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00546875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1032.6,
|
||
|
|
"completions/mean_length": 129.9931640625,
|
||
|
|
"completions/mean_terminated_length": 122.26376342773438,
|
||
|
|
"completions/min_length": 33.2,
|
||
|
|
"completions/min_terminated_length": 33.2,
|
||
|
|
"epoch": 0.112,
|
||
|
|
"grad_norm": 0.0019039362668991089,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0101,
|
||
|
|
"num_tokens": 124885633.0,
|
||
|
|
"reward": 0.9506356954574585,
|
||
|
|
"reward_std": 0.16324757039546967,
|
||
|
|
"rewards/accuracy_reward": 0.23447265625,
|
||
|
|
"rewards/brier_reward": 0.7872580051422119,
|
||
|
|
"rewards/confidence_one_or_zero": 0.47919921875,
|
||
|
|
"rewards/format_reward": 0.99111328125,
|
||
|
|
"rewards/mean_confidence_reward": 0.1406856507062912,
|
||
|
|
"rewards/smce": -0.11157373487949371,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0017578125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 831.4,
|
||
|
|
"completions/mean_length": 122.6791015625,
|
||
|
|
"completions/mean_terminated_length": 120.19132690429687,
|
||
|
|
"completions/min_length": 33.2,
|
||
|
|
"completions/min_terminated_length": 33.2,
|
||
|
|
"epoch": 0.128,
|
||
|
|
"grad_norm": 0.001967265736311674,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.003,
|
||
|
|
"num_tokens": 141058539.0,
|
||
|
|
"reward": 0.9794683218002319,
|
||
|
|
"reward_std": 0.14601433873176575,
|
||
|
|
"rewards/accuracy_reward": 0.20146484375,
|
||
|
|
"rewards/brier_reward": 0.8251733541488647,
|
||
|
|
"rewards/confidence_one_or_zero": 0.3142578125,
|
||
|
|
"rewards/format_reward": 0.99560546875,
|
||
|
|
"rewards/mean_confidence_reward": 0.16780101656913757,
|
||
|
|
"rewards/smce": -0.06330799236893654,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0015625,
|
||
|
|
"completions/max_length": 1332.0,
|
||
|
|
"completions/max_terminated_length": 777.4,
|
||
|
|
"completions/mean_length": 121.993359375,
|
||
|
|
"completions/mean_terminated_length": 119.779833984375,
|
||
|
|
"completions/min_length": 39.8,
|
||
|
|
"completions/min_terminated_length": 39.8,
|
||
|
|
"epoch": 0.144,
|
||
|
|
"grad_norm": 0.002123710699379444,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0021,
|
||
|
|
"num_tokens": 157258183.0,
|
||
|
|
"reward": 1.0028710842132569,
|
||
|
|
"reward_std": 0.19501619338989257,
|
||
|
|
"rewards/accuracy_reward": 0.20185546875,
|
||
|
|
"rewards/brier_reward": 0.8341439127922058,
|
||
|
|
"rewards/confidence_one_or_zero": 0.13662109375,
|
||
|
|
"rewards/format_reward": 0.99736328125,
|
||
|
|
"rewards/mean_confidence_reward": 0.21607398390769958,
|
||
|
|
"rewards/smce": -0.027621227875351907,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00107421875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 457.8,
|
||
|
|
"completions/mean_length": 123.31552734375,
|
||
|
|
"completions/mean_terminated_length": 121.7956298828125,
|
||
|
|
"completions/min_length": 44.4,
|
||
|
|
"completions/min_terminated_length": 44.4,
|
||
|
|
"epoch": 0.16,
|
||
|
|
"grad_norm": 0.0010736105032265186,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"num_tokens": 173541862.0,
|
||
|
|
"reward": 1.0139008045196534,
|
||
|
|
"reward_std": 0.13272683173418046,
|
||
|
|
"rewards/accuracy_reward": 0.22783203125,
|
||
|
|
"rewards/brier_reward": 0.8250125885009766,
|
||
|
|
"rewards/confidence_one_or_zero": 0.058203125,
|
||
|
|
"rewards/format_reward": 0.9982421875,
|
||
|
|
"rewards/mean_confidence_reward": 0.22021350860595704,
|
||
|
|
"rewards/smce": -0.023285704664885998,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 387.0,
|
||
|
|
"eval_completions/max_terminated_length": 387.0,
|
||
|
|
"eval_completions/mean_length": 122.21080780029297,
|
||
|
|
"eval_completions/mean_terminated_length": 122.21080780029297,
|
||
|
|
"eval_completions/min_length": 64.0,
|
||
|
|
"eval_completions/min_terminated_length": 64.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 173541862.0,
|
||
|
|
"eval_reward": 0.9996126294136047,
|
||
|
|
"eval_reward_std": 0.19382363185286522,
|
||
|
|
"eval_rewards/accuracy_reward": 0.19921875,
|
||
|
|
"eval_rewards/brier_reward": 0.8393352031707764,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.04296875,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.2296484410762787,
|
||
|
|
"eval_rewards/smce": -0.03932909946888685,
|
||
|
|
"eval_runtime": 12.2354,
|
||
|
|
"eval_samples_per_second": 40.865,
|
||
|
|
"eval_steps_per_second": 0.163,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1251.8,
|
||
|
|
"completions/max_terminated_length": 583.6,
|
||
|
|
"completions/mean_length": 121.42333984375,
|
||
|
|
"completions/mean_terminated_length": 120.73204345703125,
|
||
|
|
"completions/min_length": 44.6,
|
||
|
|
"completions/min_terminated_length": 44.6,
|
||
|
|
"epoch": 0.176,
|
||
|
|
"grad_norm": 0.001155526377260685,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 190022357.0,
|
||
|
|
"reward": 1.0153558492660522,
|
||
|
|
"reward_std": 0.12880419939756393,
|
||
|
|
"rewards/accuracy_reward": 0.24404296875,
|
||
|
|
"rewards/brier_reward": 0.8160562515258789,
|
||
|
|
"rewards/confidence_one_or_zero": 0.02265625,
|
||
|
|
"rewards/format_reward": 0.9990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.2302953004837036,
|
||
|
|
"rewards/smce": -0.02841135747730732,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1041.8,
|
||
|
|
"completions/max_terminated_length": 362.0,
|
||
|
|
"completions/mean_length": 119.63798828125,
|
||
|
|
"completions/mean_terminated_length": 118.9463104248047,
|
||
|
|
"completions/min_length": 45.6,
|
||
|
|
"completions/min_terminated_length": 45.6,
|
||
|
|
"epoch": 0.192,
|
||
|
|
"grad_norm": 0.0021765059791505337,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 206062266.0,
|
||
|
|
"reward": 1.0223777770996094,
|
||
|
|
"reward_std": 0.1203025907278061,
|
||
|
|
"rewards/accuracy_reward": 0.28935546875,
|
||
|
|
"rewards/brier_reward": 0.7992676138877869,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00576171875,
|
||
|
|
"rewards/format_reward": 0.99873046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.27211748361587523,
|
||
|
|
"rewards/smce": -0.042598482221364975,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 916.4,
|
||
|
|
"completions/max_terminated_length": 529.2,
|
||
|
|
"completions/mean_length": 120.6912109375,
|
||
|
|
"completions/mean_terminated_length": 120.41455688476563,
|
||
|
|
"completions/min_length": 57.0,
|
||
|
|
"completions/min_terminated_length": 57.0,
|
||
|
|
"epoch": 0.208,
|
||
|
|
"grad_norm": 0.0020183834712952375,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 222330368.0,
|
||
|
|
"reward": 1.0374894618988038,
|
||
|
|
"reward_std": 0.17807672172784805,
|
||
|
|
"rewards/accuracy_reward": 0.31611328125,
|
||
|
|
"rewards/brier_reward": 0.788511848449707,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0029296875,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3276663362979889,
|
||
|
|
"rewards/smce": -0.02906099855899811,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 801.8,
|
||
|
|
"completions/max_terminated_length": 576.8,
|
||
|
|
"completions/mean_length": 118.653515625,
|
||
|
|
"completions/mean_terminated_length": 118.5151382446289,
|
||
|
|
"completions/min_length": 57.0,
|
||
|
|
"completions/min_terminated_length": 57.0,
|
||
|
|
"epoch": 0.224,
|
||
|
|
"grad_norm": 0.0018503839382901788,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 238698564.0,
|
||
|
|
"reward": 1.0381819605827332,
|
||
|
|
"reward_std": 0.19111664593219757,
|
||
|
|
"rewards/accuracy_reward": 0.318359375,
|
||
|
|
"rewards/brier_reward": 0.7878335952758789,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00166015625,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.33313034772872924,
|
||
|
|
"rewards/smce": -0.029536668956279755,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 409.8,
|
||
|
|
"completions/mean_length": 118.90859375,
|
||
|
|
"completions/mean_terminated_length": 118.07798919677734,
|
||
|
|
"completions/min_length": 56.8,
|
||
|
|
"completions/min_terminated_length": 56.8,
|
||
|
|
"epoch": 0.24,
|
||
|
|
"grad_norm": 0.0020928089506924152,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 255167868.0,
|
||
|
|
"reward": 1.0354205131530763,
|
||
|
|
"reward_std": 0.1417662888765335,
|
||
|
|
"rewards/accuracy_reward": 0.3701171875,
|
||
|
|
"rewards/brier_reward": 0.7690717101097106,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0005859375,
|
||
|
|
"rewards/format_reward": 0.9990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.32316468358039857,
|
||
|
|
"rewards/smce": -0.06737189423292875,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 862.4,
|
||
|
|
"completions/max_terminated_length": 383.6,
|
||
|
|
"completions/mean_length": 116.2146484375,
|
||
|
|
"completions/mean_terminated_length": 115.93745422363281,
|
||
|
|
"completions/min_length": 56.0,
|
||
|
|
"completions/min_terminated_length": 56.0,
|
||
|
|
"epoch": 0.256,
|
||
|
|
"grad_norm": 0.0021475371904671192,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 271412722.0,
|
||
|
|
"reward": 1.039701771736145,
|
||
|
|
"reward_std": 0.1479529157280922,
|
||
|
|
"rewards/accuracy_reward": 0.3443359375,
|
||
|
|
"rewards/brier_reward": 0.7795972108840943,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.331466943025589,
|
||
|
|
"rewards/smce": -0.04423721497878432,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 804.6,
|
||
|
|
"completions/max_terminated_length": 404.6,
|
||
|
|
"completions/mean_length": 118.06962890625,
|
||
|
|
"completions/mean_terminated_length": 117.7927017211914,
|
||
|
|
"completions/min_length": 59.0,
|
||
|
|
"completions/min_terminated_length": 59.0,
|
||
|
|
"epoch": 0.272,
|
||
|
|
"grad_norm": 0.0012407461181282997,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 287587451.0,
|
||
|
|
"reward": 1.0504082918167115,
|
||
|
|
"reward_std": 0.10368214398622513,
|
||
|
|
"rewards/accuracy_reward": 0.36962890625,
|
||
|
|
"rewards/brier_reward": 0.7729404926300049,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3405745208263397,
|
||
|
|
"rewards/smce": -0.04136282093822956,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 477.0,
|
||
|
|
"completions/max_terminated_length": 477.0,
|
||
|
|
"completions/mean_length": 119.33857421875,
|
||
|
|
"completions/mean_terminated_length": 119.33857421875,
|
||
|
|
"completions/min_length": 57.8,
|
||
|
|
"completions/min_terminated_length": 57.8,
|
||
|
|
"epoch": 0.288,
|
||
|
|
"grad_norm": 0.001116760540753603,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0007,
|
||
|
|
"num_tokens": 303767654.0,
|
||
|
|
"reward": 1.0494994163513183,
|
||
|
|
"reward_std": 0.13718811124563218,
|
||
|
|
"rewards/accuracy_reward": 0.365625,
|
||
|
|
"rewards/brier_reward": 0.7756176590919495,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.38375293016433715,
|
||
|
|
"rewards/smce": -0.04204930327832699,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 627.4,
|
||
|
|
"completions/max_terminated_length": 377.4,
|
||
|
|
"completions/mean_length": 127.903125,
|
||
|
|
"completions/mean_terminated_length": 127.76566162109376,
|
||
|
|
"completions/min_length": 61.2,
|
||
|
|
"completions/min_terminated_length": 61.2,
|
||
|
|
"epoch": 0.304,
|
||
|
|
"grad_norm": 0.002396478084847331,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 320007334.0,
|
||
|
|
"reward": 1.0447796821594237,
|
||
|
|
"reward_std": 0.22138679772615433,
|
||
|
|
"rewards/accuracy_reward": 0.36904296875,
|
||
|
|
"rewards/brier_reward": 0.7709169507026672,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.41757816076278687,
|
||
|
|
"rewards/smce": -0.05001077130436897,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 830.0,
|
||
|
|
"completions/max_terminated_length": 377.4,
|
||
|
|
"completions/mean_length": 137.03095703125,
|
||
|
|
"completions/mean_terminated_length": 136.7580078125,
|
||
|
|
"completions/min_length": 59.8,
|
||
|
|
"completions/min_terminated_length": 59.8,
|
||
|
|
"epoch": 0.32,
|
||
|
|
"grad_norm": 0.0017995622474700212,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 336499235.0,
|
||
|
|
"reward": 1.0546803951263428,
|
||
|
|
"reward_std": 0.13031208738684655,
|
||
|
|
"rewards/accuracy_reward": 0.39091796875,
|
||
|
|
"rewards/brier_reward": 0.7717086434364319,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.40168737173080443,
|
||
|
|
"rewards/smce": -0.05287594236433506,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 249.0,
|
||
|
|
"eval_completions/max_terminated_length": 249.0,
|
||
|
|
"eval_completions/mean_length": 141.00047302246094,
|
||
|
|
"eval_completions/mean_terminated_length": 141.00047302246094,
|
||
|
|
"eval_completions/min_length": 71.5,
|
||
|
|
"eval_completions/min_terminated_length": 71.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 336499235.0,
|
||
|
|
"eval_reward": 1.008398711681366,
|
||
|
|
"eval_reward_std": 0.4021185487508774,
|
||
|
|
"eval_rewards/accuracy_reward": 0.3046875,
|
||
|
|
"eval_rewards/brier_reward": 0.7937804758548737,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.38554689288139343,
|
||
|
|
"eval_rewards/smce": -0.08167129755020142,
|
||
|
|
"eval_runtime": 9.8627,
|
||
|
|
"eval_samples_per_second": 50.696,
|
||
|
|
"eval_steps_per_second": 0.203,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 625.2,
|
||
|
|
"completions/max_terminated_length": 418.2,
|
||
|
|
"completions/mean_length": 143.5283203125,
|
||
|
|
"completions/mean_terminated_length": 143.39237060546876,
|
||
|
|
"completions/min_length": 62.2,
|
||
|
|
"completions/min_terminated_length": 62.2,
|
||
|
|
"epoch": 0.336,
|
||
|
|
"grad_norm": 0.0010556376073509455,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 352691397.0,
|
||
|
|
"reward": 1.0556965589523315,
|
||
|
|
"reward_std": 0.156609907746315,
|
||
|
|
"rewards/accuracy_reward": 0.39580078125,
|
||
|
|
"rewards/brier_reward": 0.7704579830169678,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.38710381984710696,
|
||
|
|
"rewards/smce": -0.05476872548460961,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 913.0,
|
||
|
|
"completions/max_terminated_length": 505.8,
|
||
|
|
"completions/mean_length": 152.359765625,
|
||
|
|
"completions/mean_terminated_length": 152.0894989013672,
|
||
|
|
"completions/min_length": 68.8,
|
||
|
|
"completions/min_terminated_length": 68.8,
|
||
|
|
"epoch": 0.352,
|
||
|
|
"grad_norm": 0.0008620854932814837,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 369511977.0,
|
||
|
|
"reward": 1.053104543685913,
|
||
|
|
"reward_std": 0.12423949837684631,
|
||
|
|
"rewards/accuracy_reward": 0.34990234375,
|
||
|
|
"rewards/brier_reward": 0.7807921409606934,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3569404065608978,
|
||
|
|
"rewards/smce": -0.024290787056088448,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1111.4,
|
||
|
|
"completions/max_terminated_length": 436.0,
|
||
|
|
"completions/mean_length": 158.96328125,
|
||
|
|
"completions/mean_terminated_length": 158.55966796875,
|
||
|
|
"completions/min_length": 68.4,
|
||
|
|
"completions/min_terminated_length": 68.4,
|
||
|
|
"epoch": 0.368,
|
||
|
|
"grad_norm": 0.0018711697775870562,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 386205233.0,
|
||
|
|
"reward": 1.0552223086357118,
|
||
|
|
"reward_std": 0.15189102590084075,
|
||
|
|
"rewards/accuracy_reward": 0.3724609375,
|
||
|
|
"rewards/brier_reward": 0.781423008441925,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.377591472864151,
|
||
|
|
"rewards/smce": -0.042951859347522256,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 927.2,
|
||
|
|
"completions/max_terminated_length": 491.4,
|
||
|
|
"completions/mean_length": 164.28662109375,
|
||
|
|
"completions/mean_terminated_length": 164.01817932128907,
|
||
|
|
"completions/min_length": 70.0,
|
||
|
|
"completions/min_terminated_length": 70.0,
|
||
|
|
"epoch": 0.384,
|
||
|
|
"grad_norm": 0.0008394431788474321,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 402744040.0,
|
||
|
|
"reward": 1.070815658569336,
|
||
|
|
"reward_std": 0.13146054446697236,
|
||
|
|
"rewards/accuracy_reward": 0.3994140625,
|
||
|
|
"rewards/brier_reward": 0.7760649085044861,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.39715608954429626,
|
||
|
|
"rewards/smce": -0.033653135597705844,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 975.0,
|
||
|
|
"completions/max_terminated_length": 543.0,
|
||
|
|
"completions/mean_length": 166.998046875,
|
||
|
|
"completions/mean_terminated_length": 166.59744262695312,
|
||
|
|
"completions/min_length": 73.8,
|
||
|
|
"completions/min_terminated_length": 73.8,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"grad_norm": 0.0017887396970763803,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 419490548.0,
|
||
|
|
"reward": 1.0607719659805297,
|
||
|
|
"reward_std": 0.19124504327774047,
|
||
|
|
"rewards/accuracy_reward": 0.37705078125,
|
||
|
|
"rewards/brier_reward": 0.7800614953041076,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4110954165458679,
|
||
|
|
"rewards/smce": -0.03517847005277872,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1126.4,
|
||
|
|
"completions/max_terminated_length": 536.4,
|
||
|
|
"completions/mean_length": 168.71416015625,
|
||
|
|
"completions/mean_terminated_length": 168.04581298828126,
|
||
|
|
"completions/min_length": 73.8,
|
||
|
|
"completions/min_terminated_length": 73.8,
|
||
|
|
"epoch": 0.416,
|
||
|
|
"grad_norm": 0.0007728107739239931,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 436099365.0,
|
||
|
|
"reward": 1.0683367729187012,
|
||
|
|
"reward_std": 0.10274968966841698,
|
||
|
|
"rewards/accuracy_reward": 0.38662109375,
|
||
|
|
"rewards/brier_reward": 0.7777512311935425,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3897749066352844,
|
||
|
|
"rewards/smce": -0.027113689482212065,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1134.6,
|
||
|
|
"completions/max_terminated_length": 493.4,
|
||
|
|
"completions/mean_length": 170.27578125,
|
||
|
|
"completions/mean_terminated_length": 169.74155883789064,
|
||
|
|
"completions/min_length": 71.4,
|
||
|
|
"completions/min_terminated_length": 71.4,
|
||
|
|
"epoch": 0.432,
|
||
|
|
"grad_norm": 0.000608608650509268,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 452857325.0,
|
||
|
|
"reward": 1.0728872060775756,
|
||
|
|
"reward_std": 0.11906236484646797,
|
||
|
|
"rewards/accuracy_reward": 0.41728515625,
|
||
|
|
"rewards/brier_reward": 0.77127103805542,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.4053739070892334,
|
||
|
|
"rewards/smce": -0.04219664148986339,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 614.4,
|
||
|
|
"completions/max_terminated_length": 391.6,
|
||
|
|
"completions/mean_length": 173.2181640625,
|
||
|
|
"completions/mean_terminated_length": 173.08515014648438,
|
||
|
|
"completions/min_length": 78.0,
|
||
|
|
"completions/min_terminated_length": 78.0,
|
||
|
|
"epoch": 0.448,
|
||
|
|
"grad_norm": 0.0007781169260852039,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 469583879.0,
|
||
|
|
"reward": 1.0613349199295044,
|
||
|
|
"reward_std": 0.19281143099069595,
|
||
|
|
"rewards/accuracy_reward": 0.39814453125,
|
||
|
|
"rewards/brier_reward": 0.7739669680595398,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4190573275089264,
|
||
|
|
"rewards/smce": -0.04924714155495167,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 716.8,
|
||
|
|
"completions/max_terminated_length": 509.6,
|
||
|
|
"completions/mean_length": 176.6033203125,
|
||
|
|
"completions/mean_terminated_length": 176.33763732910157,
|
||
|
|
"completions/min_length": 75.6,
|
||
|
|
"completions/min_terminated_length": 75.6,
|
||
|
|
"epoch": 0.464,
|
||
|
|
"grad_norm": 0.001610193052329123,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 486563113.0,
|
||
|
|
"reward": 1.0416291236877442,
|
||
|
|
"reward_std": 0.17467037439346314,
|
||
|
|
"rewards/accuracy_reward": 0.3498046875,
|
||
|
|
"rewards/brier_reward": 0.7884636282920837,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.3908169031143188,
|
||
|
|
"rewards/smce": -0.05471789538860321,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1091.6,
|
||
|
|
"completions/max_terminated_length": 458.8,
|
||
|
|
"completions/mean_length": 173.646875,
|
||
|
|
"completions/mean_terminated_length": 173.1146270751953,
|
||
|
|
"completions/min_length": 77.8,
|
||
|
|
"completions/min_terminated_length": 77.8,
|
||
|
|
"epoch": 0.48,
|
||
|
|
"grad_norm": 0.0008457148214802146,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 503389289.0,
|
||
|
|
"reward": 1.0622626066207885,
|
||
|
|
"reward_std": 0.0947317972779274,
|
||
|
|
"rewards/accuracy_reward": 0.4201171875,
|
||
|
|
"rewards/brier_reward": 0.7670642733573914,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3603034496307373,
|
||
|
|
"rewards/smce": -0.06207103803753853,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 346.0,
|
||
|
|
"eval_completions/max_terminated_length": 346.0,
|
||
|
|
"eval_completions/mean_length": 174.4136734008789,
|
||
|
|
"eval_completions/mean_terminated_length": 174.4136734008789,
|
||
|
|
"eval_completions/min_length": 79.5,
|
||
|
|
"eval_completions/min_terminated_length": 79.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 503389289.0,
|
||
|
|
"eval_reward": 1.0325101017951965,
|
||
|
|
"eval_reward_std": 0.2338552139699459,
|
||
|
|
"eval_rewards/accuracy_reward": 0.322265625,
|
||
|
|
"eval_rewards/brier_reward": 0.7960498034954071,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.3545898497104645,
|
||
|
|
"eval_rewards/smce": -0.05329589918255806,
|
||
|
|
"eval_runtime": 11.9083,
|
||
|
|
"eval_samples_per_second": 41.987,
|
||
|
|
"eval_steps_per_second": 0.168,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 952.4,
|
||
|
|
"completions/max_terminated_length": 502.4,
|
||
|
|
"completions/mean_length": 171.3103515625,
|
||
|
|
"completions/mean_terminated_length": 171.0440460205078,
|
||
|
|
"completions/min_length": 75.6,
|
||
|
|
"completions/min_terminated_length": 75.6,
|
||
|
|
"epoch": 0.496,
|
||
|
|
"grad_norm": 0.0007915415917523205,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 520451347.0,
|
||
|
|
"reward": 1.0678793430328368,
|
||
|
|
"reward_std": 0.06508489176630974,
|
||
|
|
"rewards/accuracy_reward": 0.405859375,
|
||
|
|
"rewards/brier_reward": 0.7695850133895874,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3678733766078949,
|
||
|
|
"rewards/smce": -0.03929574340581894,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 679.6,
|
||
|
|
"completions/max_terminated_length": 461.0,
|
||
|
|
"completions/mean_length": 168.5634765625,
|
||
|
|
"completions/mean_terminated_length": 168.43018493652343,
|
||
|
|
"completions/min_length": 66.6,
|
||
|
|
"completions/min_terminated_length": 66.6,
|
||
|
|
"epoch": 0.512,
|
||
|
|
"grad_norm": 0.001805165666155517,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 537323101.0,
|
||
|
|
"reward": 1.0813367128372193,
|
||
|
|
"reward_std": 0.12635424956679345,
|
||
|
|
"rewards/accuracy_reward": 0.42490234375,
|
||
|
|
"rewards/brier_reward": 0.7725860834121704,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.42473828196525576,
|
||
|
|
"rewards/smce": -0.034718182310462,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 899.2,
|
||
|
|
"completions/max_terminated_length": 467.4,
|
||
|
|
"completions/mean_length": 168.34658203125,
|
||
|
|
"completions/mean_terminated_length": 168.0794708251953,
|
||
|
|
"completions/min_length": 75.2,
|
||
|
|
"completions/min_terminated_length": 75.2,
|
||
|
|
"epoch": 0.528,
|
||
|
|
"grad_norm": 0.0018557666335254908,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 554076506.0,
|
||
|
|
"reward": 1.0747482538223267,
|
||
|
|
"reward_std": 0.21446263790130615,
|
||
|
|
"rewards/accuracy_reward": 0.4189453125,
|
||
|
|
"rewards/brier_reward": 0.7756845116615295,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4623222589492798,
|
||
|
|
"rewards/smce": -0.04493898786604404,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1087.8,
|
||
|
|
"completions/max_terminated_length": 410.4,
|
||
|
|
"completions/mean_length": 167.5201171875,
|
||
|
|
"completions/mean_terminated_length": 166.85220031738282,
|
||
|
|
"completions/min_length": 71.0,
|
||
|
|
"completions/min_terminated_length": 71.0,
|
||
|
|
"epoch": 0.544,
|
||
|
|
"grad_norm": 0.0006599732441827655,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"num_tokens": 570955496.0,
|
||
|
|
"reward": 1.0958893299102783,
|
||
|
|
"reward_std": 0.132136981934309,
|
||
|
|
"rewards/accuracy_reward": 0.45107421875,
|
||
|
|
"rewards/brier_reward": 0.7667140364646912,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99931640625,
|
||
|
|
"rewards/mean_confidence_reward": 0.4455615162849426,
|
||
|
|
"rewards/smce": -0.025326891243457793,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 863.8,
|
||
|
|
"completions/max_terminated_length": 435.8,
|
||
|
|
"completions/mean_length": 169.76455078125,
|
||
|
|
"completions/mean_terminated_length": 169.23124084472656,
|
||
|
|
"completions/min_length": 75.6,
|
||
|
|
"completions/min_terminated_length": 75.6,
|
||
|
|
"epoch": 0.56,
|
||
|
|
"grad_norm": 0.0016425641952082515,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 587515293.0,
|
||
|
|
"reward": 1.0665081977844237,
|
||
|
|
"reward_std": 0.18351908326148986,
|
||
|
|
"rewards/accuracy_reward": 0.39267578125,
|
||
|
|
"rewards/brier_reward": 0.7788697123527527,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.423252534866333,
|
||
|
|
"rewards/smce": -0.038139309734106064,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 866.4,
|
||
|
|
"completions/max_terminated_length": 654.8,
|
||
|
|
"completions/mean_length": 169.991796875,
|
||
|
|
"completions/mean_terminated_length": 169.85823974609374,
|
||
|
|
"completions/min_length": 75.4,
|
||
|
|
"completions/min_terminated_length": 75.4,
|
||
|
|
"epoch": 0.576,
|
||
|
|
"grad_norm": 0.0007072224980220199,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 604442633.0,
|
||
|
|
"reward": 1.061542308330536,
|
||
|
|
"reward_std": 0.07586740329861641,
|
||
|
|
"rewards/accuracy_reward": 0.39912109375,
|
||
|
|
"rewards/brier_reward": 0.7757653713226318,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.38846516609191895,
|
||
|
|
"rewards/smce": -0.0517049060203135,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 490.0,
|
||
|
|
"completions/max_terminated_length": 490.0,
|
||
|
|
"completions/mean_length": 169.1283203125,
|
||
|
|
"completions/mean_terminated_length": 169.1283203125,
|
||
|
|
"completions/min_length": 78.0,
|
||
|
|
"completions/min_terminated_length": 78.0,
|
||
|
|
"epoch": 0.592,
|
||
|
|
"grad_norm": 0.0006896816776134074,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 621342219.0,
|
||
|
|
"reward": 1.0860573053359985,
|
||
|
|
"reward_std": 0.07837061062455178,
|
||
|
|
"rewards/accuracy_reward": 0.42060546875,
|
||
|
|
"rewards/brier_reward": 0.775849437713623,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4021332383155823,
|
||
|
|
"rewards/smce": -0.024243370443582535,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 855.4,
|
||
|
|
"completions/max_terminated_length": 419.8,
|
||
|
|
"completions/mean_length": 169.9259765625,
|
||
|
|
"completions/mean_terminated_length": 169.65957641601562,
|
||
|
|
"completions/min_length": 66.6,
|
||
|
|
"completions/min_terminated_length": 66.6,
|
||
|
|
"epoch": 0.608,
|
||
|
|
"grad_norm": 0.0015429784543812275,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 638081749.0,
|
||
|
|
"reward": 1.078372049331665,
|
||
|
|
"reward_std": 0.14375375732779502,
|
||
|
|
"rewards/accuracy_reward": 0.41201171875,
|
||
|
|
"rewards/brier_reward": 0.7825367450714111,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4279746115207672,
|
||
|
|
"rewards/smce": -0.037512218952178954,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 737.8,
|
||
|
|
"completions/max_terminated_length": 545.6,
|
||
|
|
"completions/mean_length": 171.880078125,
|
||
|
|
"completions/mean_terminated_length": 171.74692993164064,
|
||
|
|
"completions/min_length": 79.0,
|
||
|
|
"completions/min_terminated_length": 79.0,
|
||
|
|
"epoch": 0.624,
|
||
|
|
"grad_norm": 0.001811556750908494,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 655185705.0,
|
||
|
|
"reward": 1.0680161714553833,
|
||
|
|
"reward_std": 0.1930210031569004,
|
||
|
|
"rewards/accuracy_reward": 0.40322265625,
|
||
|
|
"rewards/brier_reward": 0.7761505842208862,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.44433175325393676,
|
||
|
|
"rewards/smce": -0.04324411116540432,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 867.0,
|
||
|
|
"completions/max_terminated_length": 457.2,
|
||
|
|
"completions/mean_length": 172.5982421875,
|
||
|
|
"completions/mean_terminated_length": 172.331591796875,
|
||
|
|
"completions/min_length": 77.6,
|
||
|
|
"completions/min_terminated_length": 77.6,
|
||
|
|
"epoch": 0.64,
|
||
|
|
"grad_norm": 0.0007256731623783708,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 672295799.0,
|
||
|
|
"reward": 1.0860810041427613,
|
||
|
|
"reward_std": 0.12469884678721428,
|
||
|
|
"rewards/accuracy_reward": 0.45810546875,
|
||
|
|
"rewards/brier_reward": 0.7743456363677979,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4450886070728302,
|
||
|
|
"rewards/smce": -0.06009464208036661,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 317.5,
|
||
|
|
"eval_completions/max_terminated_length": 317.5,
|
||
|
|
"eval_completions/mean_length": 174.84073638916016,
|
||
|
|
"eval_completions/mean_terminated_length": 174.84073638916016,
|
||
|
|
"eval_completions/min_length": 86.5,
|
||
|
|
"eval_completions/min_terminated_length": 86.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 672295799.0,
|
||
|
|
"eval_reward": 1.032403826713562,
|
||
|
|
"eval_reward_std": 0.44501954317092896,
|
||
|
|
"eval_rewards/accuracy_reward": 0.359375,
|
||
|
|
"eval_rewards/brier_reward": 0.7793614268302917,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4321940243244171,
|
||
|
|
"eval_rewards/smce": -0.07392958551645279,
|
||
|
|
"eval_runtime": 11.2627,
|
||
|
|
"eval_samples_per_second": 44.394,
|
||
|
|
"eval_steps_per_second": 0.178,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 672.4,
|
||
|
|
"completions/max_terminated_length": 443.6,
|
||
|
|
"completions/mean_length": 171.9009765625,
|
||
|
|
"completions/mean_terminated_length": 171.76771850585936,
|
||
|
|
"completions/min_length": 72.6,
|
||
|
|
"completions/min_terminated_length": 72.6,
|
||
|
|
"epoch": 0.656,
|
||
|
|
"grad_norm": 0.0017715675057843328,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 688912609.0,
|
||
|
|
"reward": 1.0504565954208374,
|
||
|
|
"reward_std": 0.1744945839047432,
|
||
|
|
"rewards/accuracy_reward": 0.3833984375,
|
||
|
|
"rewards/brier_reward": 0.7761458277702331,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.43813673257827757,
|
||
|
|
"rewards/smce": -0.0585342787206173,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 858.0,
|
||
|
|
"completions/max_terminated_length": 403.2,
|
||
|
|
"completions/mean_length": 169.2025390625,
|
||
|
|
"completions/mean_terminated_length": 168.9353759765625,
|
||
|
|
"completions/min_length": 77.6,
|
||
|
|
"completions/min_terminated_length": 77.6,
|
||
|
|
"epoch": 0.672,
|
||
|
|
"grad_norm": 0.0013544198591262102,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 705558683.0,
|
||
|
|
"reward": 1.0809198856353759,
|
||
|
|
"reward_std": 0.07898051738739013,
|
||
|
|
"rewards/accuracy_reward": 0.42177734375,
|
||
|
|
"rewards/brier_reward": 0.7770021557807922,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.39942772388458253,
|
||
|
|
"rewards/smce": -0.03674512431025505,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 930.6,
|
||
|
|
"completions/max_terminated_length": 487.6,
|
||
|
|
"completions/mean_length": 165.88359375,
|
||
|
|
"completions/mean_terminated_length": 165.615380859375,
|
||
|
|
"completions/min_length": 72.6,
|
||
|
|
"completions/min_terminated_length": 72.6,
|
||
|
|
"epoch": 0.688,
|
||
|
|
"grad_norm": 0.0007912909495644271,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 722211251.0,
|
||
|
|
"reward": 1.0810879468917847,
|
||
|
|
"reward_std": 0.10564249902963638,
|
||
|
|
"rewards/accuracy_reward": 0.42451171875,
|
||
|
|
"rewards/brier_reward": 0.7782056927680969,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4082004904747009,
|
||
|
|
"rewards/smce": -0.040249383077025414,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 637.6,
|
||
|
|
"completions/max_terminated_length": 441.6,
|
||
|
|
"completions/mean_length": 163.289453125,
|
||
|
|
"completions/mean_terminated_length": 163.1553527832031,
|
||
|
|
"completions/min_length": 74.8,
|
||
|
|
"completions/min_terminated_length": 74.8,
|
||
|
|
"epoch": 0.704,
|
||
|
|
"grad_norm": 0.0007990578305907547,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 738749479.0,
|
||
|
|
"reward": 1.1020585775375367,
|
||
|
|
"reward_std": 0.05479091256856918,
|
||
|
|
"rewards/accuracy_reward": 0.45537109375,
|
||
|
|
"rewards/brier_reward": 0.7723845362663269,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.433248370885849,
|
||
|
|
"rewards/smce": -0.023541683424264194,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 674.6,
|
||
|
|
"completions/max_terminated_length": 523.8,
|
||
|
|
"completions/mean_length": 160.61611328125,
|
||
|
|
"completions/mean_terminated_length": 160.48172607421876,
|
||
|
|
"completions/min_length": 76.0,
|
||
|
|
"completions/min_terminated_length": 76.0,
|
||
|
|
"epoch": 0.72,
|
||
|
|
"grad_norm": 0.001754553522914648,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 755404044.0,
|
||
|
|
"reward": 1.097388005256653,
|
||
|
|
"reward_std": 0.16937963739037515,
|
||
|
|
"rewards/accuracy_reward": 0.46162109375,
|
||
|
|
"rewards/brier_reward": 0.7779244422912598,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.49155274629592893,
|
||
|
|
"rewards/smce": -0.04467286169528961,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 897.2,
|
||
|
|
"completions/max_terminated_length": 518.6,
|
||
|
|
"completions/mean_length": 159.86923828125,
|
||
|
|
"completions/mean_terminated_length": 159.6005859375,
|
||
|
|
"completions/min_length": 74.8,
|
||
|
|
"completions/min_terminated_length": 74.8,
|
||
|
|
"epoch": 0.736,
|
||
|
|
"grad_norm": 0.0007495178724639118,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 771980689.0,
|
||
|
|
"reward": 1.0950091004371643,
|
||
|
|
"reward_std": 0.15115555971860886,
|
||
|
|
"rewards/accuracy_reward": 0.46552734375,
|
||
|
|
"rewards/brier_reward": 0.7746174335479736,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5008993089199066,
|
||
|
|
"rewards/smce": -0.04963929690420628,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 610.4,
|
||
|
|
"completions/max_terminated_length": 379.0,
|
||
|
|
"completions/mean_length": 164.11552734375,
|
||
|
|
"completions/mean_terminated_length": 163.98125305175782,
|
||
|
|
"completions/min_length": 75.2,
|
||
|
|
"completions/min_terminated_length": 75.2,
|
||
|
|
"epoch": 0.752,
|
||
|
|
"grad_norm": 0.0018389074830338359,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0009,
|
||
|
|
"num_tokens": 788888432.0,
|
||
|
|
"reward": 1.0812556982040404,
|
||
|
|
"reward_std": 0.18698384910821914,
|
||
|
|
"rewards/accuracy_reward": 0.44228515625,
|
||
|
|
"rewards/brier_reward": 0.7731466650962829,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4763873994350433,
|
||
|
|
"rewards/smce": -0.05282366499304771,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 868.0,
|
||
|
|
"completions/max_terminated_length": 417.2,
|
||
|
|
"completions/mean_length": 170.9630859375,
|
||
|
|
"completions/mean_terminated_length": 170.5629150390625,
|
||
|
|
"completions/min_length": 78.0,
|
||
|
|
"completions/min_terminated_length": 78.0,
|
||
|
|
"epoch": 0.768,
|
||
|
|
"grad_norm": 0.0017652820097282529,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 805571798.0,
|
||
|
|
"reward": 1.1029137134552003,
|
||
|
|
"reward_std": 0.1058156132698059,
|
||
|
|
"rewards/accuracy_reward": 0.4453125,
|
||
|
|
"rewards/brier_reward": 0.7754810214042663,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4427620470523834,
|
||
|
|
"rewards/smce": -0.014673935808241367,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 481.2,
|
||
|
|
"completions/max_terminated_length": 481.2,
|
||
|
|
"completions/mean_length": 175.53525390625,
|
||
|
|
"completions/mean_terminated_length": 175.53525390625,
|
||
|
|
"completions/min_length": 79.8,
|
||
|
|
"completions/min_terminated_length": 79.8,
|
||
|
|
"epoch": 0.784,
|
||
|
|
"grad_norm": 0.0016631303587928414,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0002,
|
||
|
|
"num_tokens": 822543647.0,
|
||
|
|
"reward": 1.0799616813659667,
|
||
|
|
"reward_std": 0.08547558933496475,
|
||
|
|
"rewards/accuracy_reward": 0.45078125,
|
||
|
|
"rewards/brier_reward": 0.7736279129981994,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4300781190395355,
|
||
|
|
"rewards/smce": -0.06448663324117661,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 505.2,
|
||
|
|
"completions/max_terminated_length": 505.2,
|
||
|
|
"completions/mean_length": 185.081640625,
|
||
|
|
"completions/mean_terminated_length": 185.081640625,
|
||
|
|
"completions/min_length": 84.4,
|
||
|
|
"completions/min_terminated_length": 84.4,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 0.0006091457325965166,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 839449443.0,
|
||
|
|
"reward": 1.1104381084442139,
|
||
|
|
"reward_std": 0.07731564790010452,
|
||
|
|
"rewards/accuracy_reward": 0.4669921875,
|
||
|
|
"rewards/brier_reward": 0.7791252136230469,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.45222982168197634,
|
||
|
|
"rewards/smce": -0.02504670396447182,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 360.0,
|
||
|
|
"eval_completions/max_terminated_length": 360.0,
|
||
|
|
"eval_completions/mean_length": 191.81483459472656,
|
||
|
|
"eval_completions/mean_terminated_length": 191.81483459472656,
|
||
|
|
"eval_completions/min_length": 106.5,
|
||
|
|
"eval_completions/min_terminated_length": 106.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 839449443.0,
|
||
|
|
"eval_reward": 1.0259922742843628,
|
||
|
|
"eval_reward_std": 0.450839638710022,
|
||
|
|
"eval_rewards/accuracy_reward": 0.36328125,
|
||
|
|
"eval_rewards/brier_reward": 0.7874349653720856,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 0.998046875,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4586132913827896,
|
||
|
|
"eval_rewards/smce": -0.0967794917523861,
|
||
|
|
"eval_runtime": 12.4313,
|
||
|
|
"eval_samples_per_second": 40.221,
|
||
|
|
"eval_steps_per_second": 0.161,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 461.4,
|
||
|
|
"completions/max_terminated_length": 461.4,
|
||
|
|
"completions/mean_length": 188.6380859375,
|
||
|
|
"completions/mean_terminated_length": 188.6380859375,
|
||
|
|
"completions/min_length": 87.4,
|
||
|
|
"completions/min_terminated_length": 87.4,
|
||
|
|
"epoch": 0.816,
|
||
|
|
"grad_norm": 0.0016499223420396447,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0002,
|
||
|
|
"num_tokens": 856480265.0,
|
||
|
|
"reward": 1.1105870723724365,
|
||
|
|
"reward_std": 0.183387403935194,
|
||
|
|
"rewards/accuracy_reward": 0.46494140625,
|
||
|
|
"rewards/brier_reward": 0.7762531995773315,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.48135270476341246,
|
||
|
|
"rewards/smce": -0.02002142835408449,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 823.0,
|
||
|
|
"completions/max_terminated_length": 634.0,
|
||
|
|
"completions/mean_length": 202.40654296875,
|
||
|
|
"completions/mean_terminated_length": 202.27649841308593,
|
||
|
|
"completions/min_length": 97.2,
|
||
|
|
"completions/min_terminated_length": 97.2,
|
||
|
|
"epoch": 0.832,
|
||
|
|
"grad_norm": 0.001645341981202364,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0003,
|
||
|
|
"num_tokens": 873561260.0,
|
||
|
|
"reward": 1.1026814222335815,
|
||
|
|
"reward_std": 0.1493891693651676,
|
||
|
|
"rewards/accuracy_reward": 0.46220703125,
|
||
|
|
"rewards/brier_reward": 0.7856576681137085,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4738030254840851,
|
||
|
|
"rewards/smce": -0.04240508079528808,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 1017.2,
|
||
|
|
"completions/max_terminated_length": 619.2,
|
||
|
|
"completions/mean_length": 212.58662109375,
|
||
|
|
"completions/mean_terminated_length": 212.3273681640625,
|
||
|
|
"completions/min_length": 100.2,
|
||
|
|
"completions/min_terminated_length": 100.2,
|
||
|
|
"epoch": 0.848,
|
||
|
|
"grad_norm": 0.0004959266516380012,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 890752515.0,
|
||
|
|
"reward": 1.0983349561691285,
|
||
|
|
"reward_std": 0.17799243032932283,
|
||
|
|
"rewards/accuracy_reward": 0.43671875,
|
||
|
|
"rewards/brier_reward": 0.7796290874481201,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4496878385543823,
|
||
|
|
"rewards/smce": -0.019385896250605585,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1124.6,
|
||
|
|
"completions/max_terminated_length": 591.0,
|
||
|
|
"completions/mean_length": 222.808203125,
|
||
|
|
"completions/mean_terminated_length": 222.29569091796876,
|
||
|
|
"completions/min_length": 95.8,
|
||
|
|
"completions/min_terminated_length": 95.8,
|
||
|
|
"epoch": 0.864,
|
||
|
|
"grad_norm": 0.0007305259350687265,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"num_tokens": 908020887.0,
|
||
|
|
"reward": 1.1078871726989745,
|
||
|
|
"reward_std": 0.06039171889424324,
|
||
|
|
"rewards/accuracy_reward": 0.48857421875,
|
||
|
|
"rewards/brier_reward": 0.7802868604660034,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.43827933073043823,
|
||
|
|
"rewards/smce": -0.05269692949950695,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1358.4,
|
||
|
|
"completions/max_terminated_length": 706.8,
|
||
|
|
"completions/mean_length": 227.19384765625,
|
||
|
|
"completions/mean_terminated_length": 226.42540893554687,
|
||
|
|
"completions/min_length": 103.2,
|
||
|
|
"completions/min_terminated_length": 103.2,
|
||
|
|
"epoch": 0.88,
|
||
|
|
"grad_norm": 0.0006738873780705035,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0023,
|
||
|
|
"num_tokens": 925494424.0,
|
||
|
|
"reward": 1.0921721696853637,
|
||
|
|
"reward_std": 0.11363763809204101,
|
||
|
|
"rewards/accuracy_reward": 0.43515625,
|
||
|
|
"rewards/brier_reward": 0.7936494588851929,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0005859375,
|
||
|
|
"rewards/format_reward": 0.99921875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4402643144130707,
|
||
|
|
"rewards/smce": -0.043680974096059796,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 710.2,
|
||
|
|
"completions/max_terminated_length": 710.2,
|
||
|
|
"completions/mean_length": 230.319921875,
|
||
|
|
"completions/mean_terminated_length": 230.319921875,
|
||
|
|
"completions/min_length": 108.0,
|
||
|
|
"completions/min_terminated_length": 108.0,
|
||
|
|
"epoch": 0.896,
|
||
|
|
"grad_norm": 0.0014769054250791669,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 942963748.0,
|
||
|
|
"reward": 1.090329909324646,
|
||
|
|
"reward_std": 0.1920260950922966,
|
||
|
|
"rewards/accuracy_reward": 0.44501953125,
|
||
|
|
"rewards/brier_reward": 0.7832257986068726,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.47868783473968507,
|
||
|
|
"rewards/smce": -0.04748881347477436,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1057.2,
|
||
|
|
"completions/max_terminated_length": 777.2,
|
||
|
|
"completions/mean_length": 235.7724609375,
|
||
|
|
"completions/mean_terminated_length": 235.3920684814453,
|
||
|
|
"completions/min_length": 104.0,
|
||
|
|
"completions/min_terminated_length": 104.0,
|
||
|
|
"epoch": 0.912,
|
||
|
|
"grad_norm": 0.001404157723300159,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 960429354.0,
|
||
|
|
"reward": 1.0865304470062256,
|
||
|
|
"reward_std": 0.1757066637277603,
|
||
|
|
"rewards/accuracy_reward": 0.44658203125,
|
||
|
|
"rewards/brier_reward": 0.7791101217269898,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.467769855260849,
|
||
|
|
"rewards/smce": -0.05224150661379099,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 593.2,
|
||
|
|
"completions/max_terminated_length": 593.2,
|
||
|
|
"completions/mean_length": 233.7447265625,
|
||
|
|
"completions/mean_terminated_length": 233.7447265625,
|
||
|
|
"completions/min_length": 112.4,
|
||
|
|
"completions/min_terminated_length": 112.4,
|
||
|
|
"epoch": 0.928,
|
||
|
|
"grad_norm": 0.0005844378028996289,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 977849716.0,
|
||
|
|
"reward": 1.1005787611007691,
|
||
|
|
"reward_std": 0.09420083612203597,
|
||
|
|
"rewards/accuracy_reward": 0.4736328125,
|
||
|
|
"rewards/brier_reward": 0.7723412156105042,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4323707938194275,
|
||
|
|
"rewards/smce": -0.04481726624071598,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 1024.2,
|
||
|
|
"completions/max_terminated_length": 659.2,
|
||
|
|
"completions/mean_length": 238.26455078125,
|
||
|
|
"completions/mean_terminated_length": 238.01092834472655,
|
||
|
|
"completions/min_length": 105.8,
|
||
|
|
"completions/min_terminated_length": 105.8,
|
||
|
|
"epoch": 0.944,
|
||
|
|
"grad_norm": 0.0006636297912336886,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 995264969.0,
|
||
|
|
"reward": 1.0823294997215271,
|
||
|
|
"reward_std": 0.1275633230805397,
|
||
|
|
"rewards/accuracy_reward": 0.45625,
|
||
|
|
"rewards/brier_reward": 0.7816440343856812,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4385166108608246,
|
||
|
|
"rewards/smce": -0.07294286265969277,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 652.2,
|
||
|
|
"completions/max_terminated_length": 652.2,
|
||
|
|
"completions/mean_length": 240.97216796875,
|
||
|
|
"completions/mean_terminated_length": 240.97216796875,
|
||
|
|
"completions/min_length": 109.4,
|
||
|
|
"completions/min_terminated_length": 109.4,
|
||
|
|
"epoch": 0.96,
|
||
|
|
"grad_norm": 0.0013080838834866881,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0004,
|
||
|
|
"num_tokens": 1012672844.0,
|
||
|
|
"reward": 1.0975855112075805,
|
||
|
|
"reward_std": 0.16411191448569298,
|
||
|
|
"rewards/accuracy_reward": 0.43798828125,
|
||
|
|
"rewards/brier_reward": 0.7877278208732605,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.45487695932388306,
|
||
|
|
"rewards/smce": -0.03054591305553913,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.96,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 566.0,
|
||
|
|
"eval_completions/max_terminated_length": 566.0,
|
||
|
|
"eval_completions/mean_length": 250.1463165283203,
|
||
|
|
"eval_completions/mean_terminated_length": 250.1463165283203,
|
||
|
|
"eval_completions/min_length": 139.0,
|
||
|
|
"eval_completions/min_terminated_length": 139.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 1012672844.0,
|
||
|
|
"eval_reward": 1.0451514720916748,
|
||
|
|
"eval_reward_std": 0.4478505998849869,
|
||
|
|
"eval_rewards/accuracy_reward": 0.376953125,
|
||
|
|
"eval_rewards/brier_reward": 0.7926013767719269,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4547460973262787,
|
||
|
|
"eval_rewards/smce": -0.0792523492127657,
|
||
|
|
"eval_runtime": 16.3848,
|
||
|
|
"eval_samples_per_second": 30.516,
|
||
|
|
"eval_steps_per_second": 0.122,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 680.8,
|
||
|
|
"completions/mean_length": 248.82431640625,
|
||
|
|
"completions/mean_terminated_length": 248.19550476074218,
|
||
|
|
"completions/min_length": 108.4,
|
||
|
|
"completions/min_terminated_length": 108.4,
|
||
|
|
"epoch": 0.976,
|
||
|
|
"grad_norm": 0.0005834837211295962,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 1030081925.0,
|
||
|
|
"reward": 1.1110716819763184,
|
||
|
|
"reward_std": 0.10907015204429626,
|
||
|
|
"rewards/accuracy_reward": 0.45869140625,
|
||
|
|
"rewards/brier_reward": 0.7846518516540527,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0005859375,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.45255274176597593,
|
||
|
|
"rewards/smce": -0.02071247100830078,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1159.2,
|
||
|
|
"completions/max_terminated_length": 645.4,
|
||
|
|
"completions/mean_length": 248.54375,
|
||
|
|
"completions/mean_terminated_length": 247.7925231933594,
|
||
|
|
"completions/min_length": 115.6,
|
||
|
|
"completions/min_terminated_length": 115.6,
|
||
|
|
"epoch": 0.992,
|
||
|
|
"grad_norm": 0.0006400683778338134,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"num_tokens": 1047755493.0,
|
||
|
|
"reward": 1.0945802688598634,
|
||
|
|
"reward_std": 0.1581621140241623,
|
||
|
|
"rewards/accuracy_reward": 0.44541015625,
|
||
|
|
"rewards/brier_reward": 0.7835812568664551,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.4432067096233368,
|
||
|
|
"rewards/smce": -0.039245741069316865,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00031887755102039117,
|
||
|
|
"completions/max_length": 1156.0,
|
||
|
|
"completions/max_terminated_length": 673.5,
|
||
|
|
"completions/mean_length": 252.14827728271484,
|
||
|
|
"completions/mean_terminated_length": 251.73896026611328,
|
||
|
|
"completions/min_length": 110.5,
|
||
|
|
"completions/min_terminated_length": 110.5,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"num_tokens": 1054795355.0,
|
||
|
|
"reward": 1.0936598181724548,
|
||
|
|
"reward_std": 0.13521113619208336,
|
||
|
|
"rewards/accuracy_reward": 0.45751953125,
|
||
|
|
"rewards/brier_reward": 0.7825449705123901,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000244140625,
|
||
|
|
"rewards/format_reward": 0.999755859375,
|
||
|
|
"rewards/mean_confidence_reward": 0.46235597133636475,
|
||
|
|
"rewards/smce": -0.05250175856053829,
|
||
|
|
"step": 312,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.002977700178095032,
|
||
|
|
"train_runtime": 43698.358,
|
||
|
|
"train_samples_per_second": 0.458,
|
||
|
|
"train_steps_per_second": 0.007
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 312,
|
||
|
|
"num_input_tokens_seen": 1054795355,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 60,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|