2292 lines
86 KiB
JSON
2292 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.5056,
|
|
"eval_steps": 15,
|
|
"global_step": 79,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03776041666666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4021.0,
|
|
"completions/mean_length": 752.646484375,
|
|
"completions/mean_terminated_length": 621.4458618164062,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.0064,
|
|
"grad_norm": 0.1493307501077652,
|
|
"learning_rate": 0.0,
|
|
"loss": 0.003,
|
|
"num_tokens": 1691161.0,
|
|
"reward": 0.058687787503004074,
|
|
"reward_std": 0.14106734097003937,
|
|
"rewards/accuracy_reward": 0.0234375,
|
|
"rewards/brier_reward": 0.03208737075328827,
|
|
"rewards/confidence_one_or_zero": 0.014973958022892475,
|
|
"rewards/format_reward": 0.0618489570915699,
|
|
"rewards/mean_confidence_reward": 0.07261718064546585,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.04752604166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4063.0,
|
|
"completions/mean_length": 794.03125,
|
|
"completions/mean_terminated_length": 629.2713623046875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"epoch": 0.0128,
|
|
"grad_norm": 0.06845080852508545,
|
|
"learning_rate": 3.125e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 3447881.0,
|
|
"reward": 0.04341214895248413,
|
|
"reward_std": 0.1076519638299942,
|
|
"rewards/accuracy_reward": 0.01822916604578495,
|
|
"rewards/brier_reward": 0.023671874776482582,
|
|
"rewards/confidence_one_or_zero": 0.014322916977107525,
|
|
"rewards/format_reward": 0.044921875,
|
|
"rewards/mean_confidence_reward": 0.05469400808215141,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.04752604166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4039.0,
|
|
"completions/mean_length": 796.818359375,
|
|
"completions/mean_terminated_length": 632.197509765625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.0192,
|
|
"grad_norm": 0.5858274698257446,
|
|
"learning_rate": 6.25e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 5212922.0,
|
|
"reward": 0.037643879652023315,
|
|
"reward_std": 0.09786401689052582,
|
|
"rewards/accuracy_reward": 0.014973958022892475,
|
|
"rewards/brier_reward": 0.021250195801258087,
|
|
"rewards/confidence_one_or_zero": 0.010416666977107525,
|
|
"rewards/format_reward": 0.0390625,
|
|
"rewards/mean_confidence_reward": 0.04501953348517418,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4017.0,
|
|
"completions/mean_length": 720.1979370117188,
|
|
"completions/mean_terminated_length": 582.97021484375,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"epoch": 0.0256,
|
|
"grad_norm": 0.8754671812057495,
|
|
"learning_rate": 9.375000000000001e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 6842162.0,
|
|
"reward": 0.0466373972594738,
|
|
"reward_std": 0.11959769576787949,
|
|
"rewards/accuracy_reward": 0.01953125,
|
|
"rewards/brier_reward": 0.026216192170977592,
|
|
"rewards/confidence_one_or_zero": 0.009114583022892475,
|
|
"rewards/format_reward": 0.0475260429084301,
|
|
"rewards/mean_confidence_reward": 0.056102216243743896,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3919.0,
|
|
"completions/mean_length": 793.4765625,
|
|
"completions/mean_terminated_length": 631.057373046875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.032,
|
|
"grad_norm": 0.36891379952430725,
|
|
"learning_rate": 1.25e-06,
|
|
"loss": 0.0025,
|
|
"num_tokens": 8595838.0,
|
|
"reward": 0.05700520798563957,
|
|
"reward_std": 0.14131706953048706,
|
|
"rewards/accuracy_reward": 0.02473958395421505,
|
|
"rewards/brier_reward": 0.03132636845111847,
|
|
"rewards/confidence_one_or_zero": 0.01822916604578495,
|
|
"rewards/format_reward": 0.0579427070915699,
|
|
"rewards/mean_confidence_reward": 0.06945312768220901,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.048828125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4025.0,
|
|
"completions/mean_length": 806.748046875,
|
|
"completions/mean_terminated_length": 637.8953247070312,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.0384,
|
|
"grad_norm": 0.7912997007369995,
|
|
"learning_rate": 1.5625e-06,
|
|
"loss": 0.0022,
|
|
"num_tokens": 10371731.0,
|
|
"reward": 0.08284997940063477,
|
|
"reward_std": 0.18148033320903778,
|
|
"rewards/accuracy_reward": 0.0397135429084301,
|
|
"rewards/brier_reward": 0.04720820486545563,
|
|
"rewards/confidence_one_or_zero": 0.01627604104578495,
|
|
"rewards/format_reward": 0.0787760391831398,
|
|
"rewards/mean_confidence_reward": 0.09245442599058151,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05794270833333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3984.0,
|
|
"completions/mean_length": 847.4310302734375,
|
|
"completions/mean_terminated_length": 647.6226806640625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.0448,
|
|
"grad_norm": 0.667553186416626,
|
|
"learning_rate": 1.8750000000000003e-06,
|
|
"loss": 0.0105,
|
|
"num_tokens": 12211993.0,
|
|
"reward": 0.1802235245704651,
|
|
"reward_std": 0.34788259863853455,
|
|
"rewards/accuracy_reward": 0.07421875,
|
|
"rewards/brier_reward": 0.100025974214077,
|
|
"rewards/confidence_one_or_zero": 0.02604166604578495,
|
|
"rewards/format_reward": 0.1861979216337204,
|
|
"rewards/mean_confidence_reward": 0.1944400519132614,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06770833333333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4088.0,
|
|
"completions/mean_length": 875.6373901367188,
|
|
"completions/mean_terminated_length": 641.7562866210938,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.0512,
|
|
"grad_norm": 2.044018268585205,
|
|
"learning_rate": 2.1875000000000002e-06,
|
|
"loss": 0.0247,
|
|
"num_tokens": 14088156.0,
|
|
"reward": 0.3460591435432434,
|
|
"reward_std": 0.44335469603538513,
|
|
"rewards/accuracy_reward": 0.1341145783662796,
|
|
"rewards/brier_reward": 0.19406302273273468,
|
|
"rewards/confidence_one_or_zero": 0.0462239570915699,
|
|
"rewards/format_reward": 0.3639322817325592,
|
|
"rewards/mean_confidence_reward": 0.37214192748069763,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.060546875,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4004.0,
|
|
"completions/mean_length": 833.7174682617188,
|
|
"completions/mean_terminated_length": 623.4663696289062,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.0576,
|
|
"grad_norm": 0.6014160513877869,
|
|
"learning_rate": 2.5e-06,
|
|
"loss": 0.0451,
|
|
"num_tokens": 15908106.0,
|
|
"reward": 0.568272054195404,
|
|
"reward_std": 0.524324357509613,
|
|
"rewards/accuracy_reward": 0.2259114533662796,
|
|
"rewards/brier_reward": 0.32012465596199036,
|
|
"rewards/confidence_one_or_zero": 0.044921875,
|
|
"rewards/format_reward": 0.5904948115348816,
|
|
"rewards/mean_confidence_reward": 0.613479733467102,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4061.0,
|
|
"completions/mean_length": 799.541015625,
|
|
"completions/mean_terminated_length": 594.3672485351562,
|
|
"completions/min_length": 52.0,
|
|
"completions/min_terminated_length": 52.0,
|
|
"epoch": 0.064,
|
|
"grad_norm": 0.29679393768310547,
|
|
"learning_rate": 2.8125e-06,
|
|
"loss": 0.0552,
|
|
"num_tokens": 17679241.0,
|
|
"reward": 0.6848382949829102,
|
|
"reward_std": 0.45962172746658325,
|
|
"rewards/accuracy_reward": 0.2467447966337204,
|
|
"rewards/brier_reward": 0.3781251609325409,
|
|
"rewards/confidence_one_or_zero": 0.02604166604578495,
|
|
"rewards/format_reward": 0.7447916865348816,
|
|
"rewards/mean_confidence_reward": 0.7199475169181824,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03841145833333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4045.0,
|
|
"completions/mean_length": 669.44921875,
|
|
"completions/mean_terminated_length": 532.57275390625,
|
|
"completions/min_length": 45.0,
|
|
"completions/min_terminated_length": 45.0,
|
|
"epoch": 0.0704,
|
|
"grad_norm": 0.14208440482616425,
|
|
"learning_rate": 3.125e-06,
|
|
"loss": 0.0442,
|
|
"num_tokens": 19242611.0,
|
|
"reward": 0.8144549131393433,
|
|
"reward_std": 0.4142112731933594,
|
|
"rewards/accuracy_reward": 0.3033854067325592,
|
|
"rewards/brier_reward": 0.46092522144317627,
|
|
"rewards/confidence_one_or_zero": 0.02994791604578495,
|
|
"rewards/format_reward": 0.8645833134651184,
|
|
"rewards/mean_confidence_reward": 0.7613165974617004,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4004.0,
|
|
"completions/mean_length": 616.033203125,
|
|
"completions/mean_terminated_length": 474.5711364746094,
|
|
"completions/min_length": 59.0,
|
|
"completions/min_terminated_length": 59.0,
|
|
"epoch": 0.0768,
|
|
"grad_norm": 0.1293686181306839,
|
|
"learning_rate": 3.4375e-06,
|
|
"loss": 0.0368,
|
|
"num_tokens": 20718398.0,
|
|
"reward": 0.8415715098381042,
|
|
"reward_std": 0.3420800566673279,
|
|
"rewards/accuracy_reward": 0.2643229067325592,
|
|
"rewards/brier_reward": 0.49367454648017883,
|
|
"rewards/confidence_one_or_zero": 0.02669270895421505,
|
|
"rewards/format_reward": 0.9251301884651184,
|
|
"rewards/mean_confidence_reward": 0.7343945503234863,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.02213541666666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4047.0,
|
|
"completions/mean_length": 480.97723388671875,
|
|
"completions/mean_terminated_length": 399.1457824707031,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"epoch": 0.0832,
|
|
"grad_norm": 0.12741577625274658,
|
|
"learning_rate": 3.7500000000000005e-06,
|
|
"loss": 0.0215,
|
|
"num_tokens": 21988795.0,
|
|
"reward": 0.9607958793640137,
|
|
"reward_std": 0.301248162984848,
|
|
"rewards/accuracy_reward": 0.326171875,
|
|
"rewards/brier_reward": 0.6279593110084534,
|
|
"rewards/confidence_one_or_zero": 0.012369791977107525,
|
|
"rewards/format_reward": 0.9674479365348816,
|
|
"rewards/mean_confidence_reward": 0.6128655076026917,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.01953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3471.0,
|
|
"completions/mean_length": 425.1419372558594,
|
|
"completions/mean_terminated_length": 352.01727294921875,
|
|
"completions/min_length": 64.0,
|
|
"completions/min_terminated_length": 64.0,
|
|
"epoch": 0.0896,
|
|
"grad_norm": 0.3780879080295563,
|
|
"learning_rate": 4.0625000000000005e-06,
|
|
"loss": 0.0217,
|
|
"num_tokens": 23175493.0,
|
|
"reward": 1.0137799978256226,
|
|
"reward_std": 0.2486078441143036,
|
|
"rewards/accuracy_reward": 0.3489583432674408,
|
|
"rewards/brier_reward": 0.7026799321174622,
|
|
"rewards/confidence_one_or_zero": 0.008463541977107525,
|
|
"rewards/format_reward": 0.9759114384651184,
|
|
"rewards/mean_confidence_reward": 0.5032313466072083,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.009765625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4071.0,
|
|
"completions/mean_length": 432.48699951171875,
|
|
"completions/mean_terminated_length": 396.357666015625,
|
|
"completions/min_length": 70.0,
|
|
"completions/min_terminated_length": 70.0,
|
|
"epoch": 0.096,
|
|
"grad_norm": 0.2833631932735443,
|
|
"learning_rate": 4.3750000000000005e-06,
|
|
"loss": 0.0161,
|
|
"num_tokens": 24380841.0,
|
|
"reward": 1.0357553958892822,
|
|
"reward_std": 0.1919744610786438,
|
|
"rewards/accuracy_reward": 0.3665364682674408,
|
|
"rewards/brier_reward": 0.7212426066398621,
|
|
"rewards/confidence_one_or_zero": 0.0071614584885537624,
|
|
"rewards/format_reward": 0.9837239384651184,
|
|
"rewards/mean_confidence_reward": 0.3716927468776703,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.096,
|
|
"eval_completions/clipped_ratio": 0.0146484375,
|
|
"eval_completions/max_length": 3697.5,
|
|
"eval_completions/max_terminated_length": 1968.5,
|
|
"eval_completions/mean_length": 448.44681549072266,
|
|
"eval_completions/mean_terminated_length": 394.2275581359863,
|
|
"eval_completions/min_length": 110.375,
|
|
"eval_completions/min_terminated_length": 110.375,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 24380841.0,
|
|
"eval_reward": 1.0368142426013947,
|
|
"eval_reward_std": 0.19133614003658295,
|
|
"eval_rewards/accuracy_reward": 0.388671875,
|
|
"eval_rewards/brier_reward": 0.7064355462789536,
|
|
"eval_rewards/confidence_one_or_zero": 0.00390625,
|
|
"eval_rewards/format_reward": 0.978515625,
|
|
"eval_rewards/mean_confidence_reward": 0.2681640610098839,
|
|
"eval_runtime": 265.3148,
|
|
"eval_samples_per_second": 3.769,
|
|
"eval_steps_per_second": 0.03,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00846354166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3919.0,
|
|
"completions/mean_length": 418.36004638671875,
|
|
"completions/mean_terminated_length": 386.9684753417969,
|
|
"completions/min_length": 80.0,
|
|
"completions/min_terminated_length": 80.0,
|
|
"epoch": 0.1024,
|
|
"grad_norm": 0.07689929008483887,
|
|
"learning_rate": 4.6875000000000004e-06,
|
|
"loss": 0.0152,
|
|
"num_tokens": 25564434.0,
|
|
"reward": 1.0678430795669556,
|
|
"reward_std": 0.16269861161708832,
|
|
"rewards/accuracy_reward": 0.4485677182674408,
|
|
"rewards/brier_reward": 0.7007845044136047,
|
|
"rewards/confidence_one_or_zero": 0.00390625,
|
|
"rewards/format_reward": 0.986328125,
|
|
"rewards/mean_confidence_reward": 0.28589844703674316,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00520833333333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3479.0,
|
|
"completions/mean_length": 436.69140625,
|
|
"completions/mean_terminated_length": 417.53271484375,
|
|
"completions/min_length": 107.0,
|
|
"completions/min_terminated_length": 107.0,
|
|
"epoch": 0.1088,
|
|
"grad_norm": 0.058885227888822556,
|
|
"learning_rate": 5e-06,
|
|
"loss": 0.0073,
|
|
"num_tokens": 26773504.0,
|
|
"reward": 1.0608980655670166,
|
|
"reward_std": 0.137383833527565,
|
|
"rewards/accuracy_reward": 0.3971354067325592,
|
|
"rewards/brier_reward": 0.7324674725532532,
|
|
"rewards/confidence_one_or_zero": 0.0026041667442768812,
|
|
"rewards/format_reward": 0.9921875,
|
|
"rewards/mean_confidence_reward": 0.2844401001930237,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.001953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1670.0,
|
|
"completions/mean_length": 434.02606201171875,
|
|
"completions/mean_terminated_length": 426.8597717285156,
|
|
"completions/min_length": 97.0,
|
|
"completions/min_terminated_length": 97.0,
|
|
"epoch": 0.1152,
|
|
"grad_norm": 0.04554029926657677,
|
|
"learning_rate": 4.920634920634921e-06,
|
|
"loss": 0.0014,
|
|
"num_tokens": 27969752.0,
|
|
"reward": 1.1085549592971802,
|
|
"reward_std": 0.13607333600521088,
|
|
"rewards/accuracy_reward": 0.4947916567325592,
|
|
"rewards/brier_reward": 0.7242643237113953,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.998046875,
|
|
"rewards/mean_confidence_reward": 0.3483724296092987,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0032552083333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2845.0,
|
|
"completions/mean_length": 439.365234375,
|
|
"completions/mean_terminated_length": 427.4232482910156,
|
|
"completions/min_length": 118.0,
|
|
"completions/min_terminated_length": 118.0,
|
|
"epoch": 0.1216,
|
|
"grad_norm": 0.04788957163691521,
|
|
"learning_rate": 4.841269841269842e-06,
|
|
"loss": 0.0039,
|
|
"num_tokens": 29179849.0,
|
|
"reward": 1.1145256757736206,
|
|
"reward_std": 0.14436736702919006,
|
|
"rewards/accuracy_reward": 0.486328125,
|
|
"rewards/brier_reward": 0.7459700107574463,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.41516926884651184,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3361.0,
|
|
"completions/mean_length": 450.90106201171875,
|
|
"completions/mean_terminated_length": 446.14862060546875,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"epoch": 0.128,
|
|
"grad_norm": 0.05234735086560249,
|
|
"learning_rate": 4.761904761904762e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 30404737.0,
|
|
"reward": 1.1313689947128296,
|
|
"reward_std": 0.17600169777870178,
|
|
"rewards/accuracy_reward": 0.5104166865348816,
|
|
"rewards/brier_reward": 0.753613293170929,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9986979365348816,
|
|
"rewards/mean_confidence_reward": 0.4996744692325592,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3302.0,
|
|
"completions/mean_length": 460.36199951171875,
|
|
"completions/mean_terminated_length": 455.62188720703125,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"epoch": 0.1344,
|
|
"grad_norm": 0.0575055293738842,
|
|
"learning_rate": 4.682539682539683e-06,
|
|
"loss": 0.0043,
|
|
"num_tokens": 31646789.0,
|
|
"reward": 1.146083116531372,
|
|
"reward_std": 0.16491073369979858,
|
|
"rewards/accuracy_reward": 0.5423176884651184,
|
|
"rewards/brier_reward": 0.75244140625,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9973958134651184,
|
|
"rewards/mean_confidence_reward": 0.5458984375,
|
|
"step": 21
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3233.0,
|
|
"completions/mean_length": 439.54296875,
|
|
"completions/mean_terminated_length": 434.7757263183594,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"epoch": 0.1408,
|
|
"grad_norm": 0.061949778348207474,
|
|
"learning_rate": 4.603174603174604e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 32851895.0,
|
|
"reward": 1.1546767950057983,
|
|
"reward_std": 0.15218165516853333,
|
|
"rewards/accuracy_reward": 0.5533854365348816,
|
|
"rewards/brier_reward": 0.7585611939430237,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9973958134651184,
|
|
"rewards/mean_confidence_reward": 0.5486978888511658,
|
|
"step": 22
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.001953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1641.0,
|
|
"completions/mean_length": 499.6341247558594,
|
|
"completions/mean_terminated_length": 492.5962219238281,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"epoch": 0.1472,
|
|
"grad_norm": 0.05392421409487724,
|
|
"learning_rate": 4.523809523809524e-06,
|
|
"loss": 0.0031,
|
|
"num_tokens": 34160477.0,
|
|
"reward": 1.1524631977081299,
|
|
"reward_std": 0.16562673449516296,
|
|
"rewards/accuracy_reward": 0.5553385615348816,
|
|
"rewards/brier_reward": 0.75152987241745,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.998046875,
|
|
"rewards/mean_confidence_reward": 0.550585925579071,
|
|
"step": 23
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3744.0,
|
|
"completions/mean_length": 510.9681091308594,
|
|
"completions/mean_terminated_length": 506.29400634765625,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"epoch": 0.1536,
|
|
"grad_norm": 0.06393470615148544,
|
|
"learning_rate": 4.444444444444444e-06,
|
|
"loss": 0.0047,
|
|
"num_tokens": 35477316.0,
|
|
"reward": 1.1909399032592773,
|
|
"reward_std": 0.1410152167081833,
|
|
"rewards/accuracy_reward": 0.6204426884651184,
|
|
"rewards/brier_reward": 0.7640299797058105,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9973958134651184,
|
|
"rewards/mean_confidence_reward": 0.5521484613418579,
|
|
"step": 24
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3932.0,
|
|
"completions/mean_length": 544.9310302734375,
|
|
"completions/mean_terminated_length": 516.9697875976562,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"epoch": 0.16,
|
|
"grad_norm": 0.061590950936079025,
|
|
"learning_rate": 4.365079365079366e-06,
|
|
"loss": 0.0107,
|
|
"num_tokens": 36842618.0,
|
|
"reward": 1.1610541343688965,
|
|
"reward_std": 0.1644006073474884,
|
|
"rewards/accuracy_reward": 0.578125,
|
|
"rewards/brier_reward": 0.7524349093437195,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9915364384651184,
|
|
"rewards/mean_confidence_reward": 0.5852864980697632,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0032552083333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3057.0,
|
|
"completions/mean_length": 536.7025146484375,
|
|
"completions/mean_terminated_length": 525.078369140625,
|
|
"completions/min_length": 178.0,
|
|
"completions/min_terminated_length": 178.0,
|
|
"epoch": 0.1664,
|
|
"grad_norm": 0.0578559935092926,
|
|
"learning_rate": 4.2857142857142855e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 38203305.0,
|
|
"reward": 1.1484956741333008,
|
|
"reward_std": 0.15411688387393951,
|
|
"rewards/accuracy_reward": 0.552734375,
|
|
"rewards/brier_reward": 0.7481510043144226,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.596875011920929,
|
|
"step": 26
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0026041666666666297,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2030.0,
|
|
"completions/mean_length": 526.9173583984375,
|
|
"completions/mean_terminated_length": 517.5985717773438,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"epoch": 0.1728,
|
|
"grad_norm": 0.10168619453907013,
|
|
"learning_rate": 4.206349206349207e-06,
|
|
"loss": 0.0062,
|
|
"num_tokens": 39539466.0,
|
|
"reward": 1.2171251773834229,
|
|
"reward_std": 0.14634501934051514,
|
|
"rewards/accuracy_reward": 0.6653645634651184,
|
|
"rewards/brier_reward": 0.7714778780937195,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9973958134651184,
|
|
"rewards/mean_confidence_reward": 0.5986328125,
|
|
"step": 27
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0026041666666666297,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2531.0,
|
|
"completions/mean_length": 600.4609375,
|
|
"completions/mean_terminated_length": 591.334228515625,
|
|
"completions/min_length": 182.0,
|
|
"completions/min_terminated_length": 182.0,
|
|
"epoch": 0.1792,
|
|
"grad_norm": 0.11760320514440536,
|
|
"learning_rate": 4.126984126984127e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 40994942.0,
|
|
"reward": 1.1956537961959839,
|
|
"reward_std": 0.15472522377967834,
|
|
"rewards/accuracy_reward": 0.6315104365348816,
|
|
"rewards/brier_reward": 0.7636914253234863,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.5974609851837158,
|
|
"step": 28
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00455729166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3712.0,
|
|
"completions/mean_length": 624.3177490234375,
|
|
"completions/mean_terminated_length": 608.4237670898438,
|
|
"completions/min_length": 190.0,
|
|
"completions/min_terminated_length": 190.0,
|
|
"epoch": 0.1856,
|
|
"grad_norm": 0.10314124077558517,
|
|
"learning_rate": 4.047619047619048e-06,
|
|
"loss": 0.0074,
|
|
"num_tokens": 42480030.0,
|
|
"reward": 1.1957581043243408,
|
|
"reward_std": 0.15051952004432678,
|
|
"rewards/accuracy_reward": 0.6341145634651184,
|
|
"rewards/brier_reward": 0.76324862241745,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.994140625,
|
|
"rewards/mean_confidence_reward": 0.5970703363418579,
|
|
"step": 29
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00651041666666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3992.0,
|
|
"completions/mean_length": 670.0436401367188,
|
|
"completions/mean_terminated_length": 647.5930786132812,
|
|
"completions/min_length": 177.0,
|
|
"completions/min_terminated_length": 177.0,
|
|
"epoch": 0.192,
|
|
"grad_norm": 0.042782012373209,
|
|
"learning_rate": 3.968253968253968e-06,
|
|
"loss": 0.0077,
|
|
"num_tokens": 44039289.0,
|
|
"reward": 1.1695502996444702,
|
|
"reward_std": 0.1441710889339447,
|
|
"rewards/accuracy_reward": 0.5924479365348816,
|
|
"rewards/brier_reward": 0.7538021206855774,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9928385615348816,
|
|
"rewards/mean_confidence_reward": 0.5947917103767395,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.192,
|
|
"eval_completions/clipped_ratio": 0.00390625,
|
|
"eval_completions/max_length": 2866.125,
|
|
"eval_completions/max_terminated_length": 2237.375,
|
|
"eval_completions/mean_length": 661.8221969604492,
|
|
"eval_completions/mean_terminated_length": 648.3947143554688,
|
|
"eval_completions/min_length": 265.75,
|
|
"eval_completions/min_terminated_length": 265.75,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 44039289.0,
|
|
"eval_reward": 1.1933995634317398,
|
|
"eval_reward_std": 0.29131926596164703,
|
|
"eval_rewards/accuracy_reward": 0.62890625,
|
|
"eval_rewards/brier_reward": 0.7627636343240738,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 0.9951171875,
|
|
"eval_rewards/mean_confidence_reward": 0.5977539271116257,
|
|
"eval_runtime": 192.1463,
|
|
"eval_samples_per_second": 5.204,
|
|
"eval_steps_per_second": 0.042,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2854.0,
|
|
"completions/mean_length": 665.64453125,
|
|
"completions/mean_terminated_length": 652.1921997070312,
|
|
"completions/min_length": 247.0,
|
|
"completions/min_terminated_length": 247.0,
|
|
"epoch": 0.1984,
|
|
"grad_norm": 0.040110934525728226,
|
|
"learning_rate": 3.88888888888889e-06,
|
|
"loss": 0.0072,
|
|
"num_tokens": 45598223.0,
|
|
"reward": 1.2185217142105103,
|
|
"reward_std": 0.14597812294960022,
|
|
"rewards/accuracy_reward": 0.6692708134651184,
|
|
"rewards/brier_reward": 0.7716667056083679,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.5967448353767395,
|
|
"step": 31
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0032552083333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4076.0,
|
|
"completions/mean_length": 715.7357177734375,
|
|
"completions/mean_terminated_length": 704.6962280273438,
|
|
"completions/min_length": 243.0,
|
|
"completions/min_terminated_length": 243.0,
|
|
"epoch": 0.2048,
|
|
"grad_norm": 0.0372241735458374,
|
|
"learning_rate": 3.80952380952381e-06,
|
|
"loss": 0.0099,
|
|
"num_tokens": 47225889.0,
|
|
"reward": 1.1982970237731934,
|
|
"reward_std": 0.14972180128097534,
|
|
"rewards/accuracy_reward": 0.6380208134651184,
|
|
"rewards/brier_reward": 0.7644205093383789,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.994140625,
|
|
"rewards/mean_confidence_reward": 0.5944661498069763,
|
|
"step": 32
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.01041666666666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3919.0,
|
|
"completions/mean_length": 717.7728271484375,
|
|
"completions/mean_terminated_length": 682.2125244140625,
|
|
"completions/min_length": 282.0,
|
|
"completions/min_terminated_length": 282.0,
|
|
"epoch": 0.2112,
|
|
"grad_norm": 0.04480714723467827,
|
|
"learning_rate": 3.7301587301587305e-06,
|
|
"loss": 0.0136,
|
|
"num_tokens": 48859172.0,
|
|
"reward": 1.1937364339828491,
|
|
"reward_std": 0.1636783331632614,
|
|
"rewards/accuracy_reward": 0.6380208134651184,
|
|
"rewards/brier_reward": 0.7605077624320984,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9889323115348816,
|
|
"rewards/mean_confidence_reward": 0.5912760496139526,
|
|
"step": 33
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00911458333333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4079.0,
|
|
"completions/mean_length": 758.6927490234375,
|
|
"completions/mean_terminated_length": 727.9947509765625,
|
|
"completions/min_length": 224.0,
|
|
"completions/min_terminated_length": 224.0,
|
|
"epoch": 0.2176,
|
|
"grad_norm": 0.03986569866538048,
|
|
"learning_rate": 3.6507936507936507e-06,
|
|
"loss": 0.0123,
|
|
"num_tokens": 50555628.0,
|
|
"reward": 1.1963211297988892,
|
|
"reward_std": 0.15583686530590057,
|
|
"rewards/accuracy_reward": 0.6399739384651184,
|
|
"rewards/brier_reward": 0.7617707848548889,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9908854365348816,
|
|
"rewards/mean_confidence_reward": 0.5932291746139526,
|
|
"step": 34
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.01106770833333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4009.0,
|
|
"completions/mean_length": 762.216796875,
|
|
"completions/mean_terminated_length": 724.906494140625,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"epoch": 0.224,
|
|
"grad_norm": 0.04056720435619354,
|
|
"learning_rate": 3.5714285714285718e-06,
|
|
"loss": 0.0137,
|
|
"num_tokens": 52260249.0,
|
|
"reward": 1.1862528324127197,
|
|
"reward_std": 0.16809241473674774,
|
|
"rewards/accuracy_reward": 0.6276041865348816,
|
|
"rewards/brier_reward": 0.7572590708732605,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9876301884651184,
|
|
"rewards/mean_confidence_reward": 0.5930338501930237,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00716145833333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4079.0,
|
|
"completions/mean_length": 753.8600463867188,
|
|
"completions/mean_terminated_length": 729.7528076171875,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"epoch": 0.2304,
|
|
"grad_norm": 0.03567610681056976,
|
|
"learning_rate": 3.492063492063492e-06,
|
|
"loss": 0.0077,
|
|
"num_tokens": 53953866.0,
|
|
"reward": 1.2123433351516724,
|
|
"reward_std": 0.1348019540309906,
|
|
"rewards/accuracy_reward": 0.6647135615348816,
|
|
"rewards/brier_reward": 0.767773449420929,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9921875,
|
|
"rewards/mean_confidence_reward": 0.599609375,
|
|
"step": 36
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00846354166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4015.0,
|
|
"completions/mean_length": 781.5670776367188,
|
|
"completions/mean_terminated_length": 753.2757568359375,
|
|
"completions/min_length": 306.0,
|
|
"completions/min_terminated_length": 306.0,
|
|
"epoch": 0.2368,
|
|
"grad_norm": 0.039077743887901306,
|
|
"learning_rate": 3.412698412698413e-06,
|
|
"loss": 0.0106,
|
|
"num_tokens": 55686281.0,
|
|
"reward": 1.2051103115081787,
|
|
"reward_std": 0.1598556488752365,
|
|
"rewards/accuracy_reward": 0.6555989384651184,
|
|
"rewards/brier_reward": 0.7643750309944153,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.990234375,
|
|
"rewards/mean_confidence_reward": 0.6005208492279053,
|
|
"step": 37
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00651041666666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2779.0,
|
|
"completions/mean_length": 781.4739990234375,
|
|
"completions/mean_terminated_length": 759.7536010742188,
|
|
"completions/min_length": 316.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"epoch": 0.2432,
|
|
"grad_norm": 0.035907238721847534,
|
|
"learning_rate": 3.3333333333333333e-06,
|
|
"loss": 0.0091,
|
|
"num_tokens": 57420537.0,
|
|
"reward": 1.200331687927246,
|
|
"reward_std": 0.14801643788814545,
|
|
"rewards/accuracy_reward": 0.6438801884651184,
|
|
"rewards/brier_reward": 0.7639322876930237,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9928385615348816,
|
|
"rewards/mean_confidence_reward": 0.6040365099906921,
|
|
"step": 38
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00846354166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3158.0,
|
|
"completions/mean_length": 798.0794677734375,
|
|
"completions/mean_terminated_length": 769.9290771484375,
|
|
"completions/min_length": 305.0,
|
|
"completions/min_terminated_length": 305.0,
|
|
"epoch": 0.2496,
|
|
"grad_norm": 0.03557732328772545,
|
|
"learning_rate": 3.2539682539682544e-06,
|
|
"loss": 0.0102,
|
|
"num_tokens": 59177011.0,
|
|
"reward": 1.1936421394348145,
|
|
"reward_std": 0.15311554074287415,
|
|
"rewards/accuracy_reward": 0.63671875,
|
|
"rewards/brier_reward": 0.7603189945220947,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.990234375,
|
|
"rewards/mean_confidence_reward": 0.6003255248069763,
|
|
"step": 39
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2903.0,
|
|
"completions/mean_length": 799.0560302734375,
|
|
"completions/mean_terminated_length": 773.0958251953125,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"epoch": 0.256,
|
|
"grad_norm": 0.034340471029281616,
|
|
"learning_rate": 3.1746031746031746e-06,
|
|
"loss": 0.0109,
|
|
"num_tokens": 60939993.0,
|
|
"reward": 1.1955170631408691,
|
|
"reward_std": 0.15389688313007355,
|
|
"rewards/accuracy_reward": 0.6360676884651184,
|
|
"rewards/brier_reward": 0.7627668976783752,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9921875,
|
|
"rewards/mean_confidence_reward": 0.6012369990348816,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00455729166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3974.0,
|
|
"completions/mean_length": 786.3828125,
|
|
"completions/mean_terminated_length": 771.2308349609375,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"epoch": 0.2624,
|
|
"grad_norm": 0.03095524199306965,
|
|
"learning_rate": 3.0952380952380957e-06,
|
|
"loss": 0.0061,
|
|
"num_tokens": 62680749.0,
|
|
"reward": 1.2226656675338745,
|
|
"reward_std": 0.1328285038471222,
|
|
"rewards/accuracy_reward": 0.6790364384651184,
|
|
"rewards/brier_reward": 0.7721419334411621,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.994140625,
|
|
"rewards/mean_confidence_reward": 0.6034505367279053,
|
|
"step": 41
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00716145833333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3645.0,
|
|
"completions/mean_length": 764.908203125,
|
|
"completions/mean_terminated_length": 740.8806762695312,
|
|
"completions/min_length": 257.0,
|
|
"completions/min_terminated_length": 257.0,
|
|
"epoch": 0.2688,
|
|
"grad_norm": 0.03424890711903572,
|
|
"learning_rate": 3.015873015873016e-06,
|
|
"loss": 0.0059,
|
|
"num_tokens": 64388184.0,
|
|
"reward": 1.206220269203186,
|
|
"reward_std": 0.141874298453331,
|
|
"rewards/accuracy_reward": 0.654296875,
|
|
"rewards/brier_reward": 0.7659440040588379,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9921875,
|
|
"rewards/mean_confidence_reward": 0.6021484732627869,
|
|
"step": 42
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00651041666666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2884.0,
|
|
"completions/mean_length": 774.7799682617188,
|
|
"completions/mean_terminated_length": 753.0157470703125,
|
|
"completions/min_length": 256.0,
|
|
"completions/min_terminated_length": 256.0,
|
|
"epoch": 0.2752,
|
|
"grad_norm": 0.033084969967603683,
|
|
"learning_rate": 2.936507936507937e-06,
|
|
"loss": 0.0082,
|
|
"num_tokens": 66112694.0,
|
|
"reward": 1.2110347747802734,
|
|
"reward_std": 0.1414928436279297,
|
|
"rewards/accuracy_reward": 0.6614583134651184,
|
|
"rewards/brier_reward": 0.7671093344688416,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9934895634651184,
|
|
"rewards/mean_confidence_reward": 0.6049479842185974,
|
|
"step": 43
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00520833333333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3942.0,
|
|
"completions/mean_length": 746.6986083984375,
|
|
"completions/mean_terminated_length": 729.1629638671875,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"epoch": 0.2816,
|
|
"grad_norm": 0.035444434732198715,
|
|
"learning_rate": 2.8571428571428573e-06,
|
|
"loss": 0.0063,
|
|
"num_tokens": 67793423.0,
|
|
"reward": 1.2146871089935303,
|
|
"reward_std": 0.14682471752166748,
|
|
"rewards/accuracy_reward": 0.6653645634651184,
|
|
"rewards/brier_reward": 0.7698567509651184,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.994140625,
|
|
"rewards/mean_confidence_reward": 0.6050781607627869,
|
|
"step": 44
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2623.0,
|
|
"completions/mean_length": 762.8346557617188,
|
|
"completions/mean_terminated_length": 749.763427734375,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"epoch": 0.288,
|
|
"grad_norm": 0.03420567885041237,
|
|
"learning_rate": 2.7777777777777783e-06,
|
|
"loss": 0.0038,
|
|
"num_tokens": 69496561.0,
|
|
"reward": 1.1890003681182861,
|
|
"reward_std": 0.13613253831863403,
|
|
"rewards/accuracy_reward": 0.62109375,
|
|
"rewards/brier_reward": 0.760800838470459,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.6086588501930237,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.288,
|
|
"eval_completions/clipped_ratio": 0.0068359375,
|
|
"eval_completions/max_length": 3217.875,
|
|
"eval_completions/max_terminated_length": 2009.375,
|
|
"eval_completions/mean_length": 765.9082794189453,
|
|
"eval_completions/mean_terminated_length": 742.9629592895508,
|
|
"eval_completions/min_length": 305.75,
|
|
"eval_completions/min_terminated_length": 305.75,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 69496561.0,
|
|
"eval_reward": 1.2024914920330048,
|
|
"eval_reward_std": 0.2949274182319641,
|
|
"eval_rewards/accuracy_reward": 0.6474609375,
|
|
"eval_rewards/brier_reward": 0.7643456682562828,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 0.9931640625,
|
|
"eval_rewards/mean_confidence_reward": 0.607714869081974,
|
|
"eval_runtime": 214.0442,
|
|
"eval_samples_per_second": 4.672,
|
|
"eval_steps_per_second": 0.037,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0026041666666666297,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2595.0,
|
|
"completions/mean_length": 730.7044677734375,
|
|
"completions/mean_terminated_length": 721.9177856445312,
|
|
"completions/min_length": 287.0,
|
|
"completions/min_terminated_length": 287.0,
|
|
"epoch": 0.2944,
|
|
"grad_norm": 0.03395461663603783,
|
|
"learning_rate": 2.6984126984126986e-06,
|
|
"loss": 0.005,
|
|
"num_tokens": 71157715.0,
|
|
"reward": 1.2088148593902588,
|
|
"reward_std": 0.13170814514160156,
|
|
"rewards/accuracy_reward": 0.6529948115348816,
|
|
"rewards/brier_reward": 0.7678775787353516,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.6097656488418579,
|
|
"step": 46
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00520833333333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4095.0,
|
|
"completions/mean_length": 791.0983276367188,
|
|
"completions/mean_terminated_length": 773.795166015625,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"epoch": 0.3008,
|
|
"grad_norm": 0.031042389571666718,
|
|
"learning_rate": 2.6190476190476192e-06,
|
|
"loss": 0.0041,
|
|
"num_tokens": 72907882.0,
|
|
"reward": 1.2191662788391113,
|
|
"reward_std": 0.1413819044828415,
|
|
"rewards/accuracy_reward": 0.6731770634651184,
|
|
"rewards/brier_reward": 0.7710025906562805,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.994140625,
|
|
"rewards/mean_confidence_reward": 0.6098958849906921,
|
|
"step": 47
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.01041666666666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3794.0,
|
|
"completions/mean_length": 831.8197021484375,
|
|
"completions/mean_terminated_length": 797.4598999023438,
|
|
"completions/min_length": 295.0,
|
|
"completions/min_terminated_length": 295.0,
|
|
"epoch": 0.3072,
|
|
"grad_norm": 0.034108564257621765,
|
|
"learning_rate": 2.53968253968254e-06,
|
|
"loss": 0.0078,
|
|
"num_tokens": 74725765.0,
|
|
"reward": 1.1852697134017944,
|
|
"reward_std": 0.1596524566411972,
|
|
"rewards/accuracy_reward": 0.6223958134651184,
|
|
"rewards/brier_reward": 0.7585482597351074,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9895833134651184,
|
|
"rewards/mean_confidence_reward": 0.6061848998069763,
|
|
"step": 48
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0026041666666666297,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2645.0,
|
|
"completions/mean_length": 766.7916870117188,
|
|
"completions/mean_terminated_length": 758.0992431640625,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"epoch": 0.3136,
|
|
"grad_norm": 0.03339226543903351,
|
|
"learning_rate": 2.4603174603174605e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 76440685.0,
|
|
"reward": 1.216418981552124,
|
|
"reward_std": 0.14440613985061646,
|
|
"rewards/accuracy_reward": 0.6647135615348816,
|
|
"rewards/brier_reward": 0.7707161903381348,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9973958134651184,
|
|
"rewards/mean_confidence_reward": 0.6104167103767395,
|
|
"step": 49
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2890.0,
|
|
"completions/mean_length": 791.400390625,
|
|
"completions/mean_terminated_length": 778.4412231445312,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"epoch": 0.32,
|
|
"grad_norm": 0.03476458042860031,
|
|
"learning_rate": 2.380952380952381e-06,
|
|
"loss": 0.0062,
|
|
"num_tokens": 78188732.0,
|
|
"reward": 1.2055139541625977,
|
|
"reward_std": 0.14840558171272278,
|
|
"rewards/accuracy_reward": 0.6484375,
|
|
"rewards/brier_reward": 0.7664844393730164,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.6098958849906921,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0032552083333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2566.0,
|
|
"completions/mean_length": 768.0579833984375,
|
|
"completions/mean_terminated_length": 757.1893920898438,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"epoch": 0.3264,
|
|
"grad_norm": 0.03427394852042198,
|
|
"learning_rate": 2.301587301587302e-06,
|
|
"loss": 0.0043,
|
|
"num_tokens": 79899005.0,
|
|
"reward": 1.209107756614685,
|
|
"reward_std": 0.14521203935146332,
|
|
"rewards/accuracy_reward": 0.6536458134651184,
|
|
"rewards/brier_reward": 0.7678124904632568,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.611328125,
|
|
"step": 51
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2480.0,
|
|
"completions/mean_length": 786.3619995117188,
|
|
"completions/mean_terminated_length": 773.383056640625,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"epoch": 0.3328,
|
|
"grad_norm": 0.031459398567676544,
|
|
"learning_rate": 2.222222222222222e-06,
|
|
"loss": 0.005,
|
|
"num_tokens": 81644601.0,
|
|
"reward": 1.2023563385009766,
|
|
"reward_std": 0.12966501712799072,
|
|
"rewards/accuracy_reward": 0.6432291865348816,
|
|
"rewards/brier_reward": 0.7653775811195374,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.6079427003860474,
|
|
"step": 52
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0032552083333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4012.0,
|
|
"completions/mean_length": 815.41015625,
|
|
"completions/mean_terminated_length": 804.6962280273438,
|
|
"completions/min_length": 282.0,
|
|
"completions/min_terminated_length": 282.0,
|
|
"epoch": 0.3392,
|
|
"grad_norm": 0.030849505215883255,
|
|
"learning_rate": 2.1428571428571427e-06,
|
|
"loss": 0.0057,
|
|
"num_tokens": 83426895.0,
|
|
"reward": 1.2228935956954956,
|
|
"reward_std": 0.13298486173152924,
|
|
"rewards/accuracy_reward": 0.6751301884651184,
|
|
"rewards/brier_reward": 0.7738997340202332,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.6102213859558105,
|
|
"step": 53
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0032552083333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3659.0,
|
|
"completions/mean_length": 793.6959838867188,
|
|
"completions/mean_terminated_length": 782.9111328125,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"epoch": 0.3456,
|
|
"grad_norm": 0.026791466400027275,
|
|
"learning_rate": 2.0634920634920634e-06,
|
|
"loss": 0.0035,
|
|
"num_tokens": 85178092.0,
|
|
"reward": 1.236464500427246,
|
|
"reward_std": 0.10865309834480286,
|
|
"rewards/accuracy_reward": 0.697265625,
|
|
"rewards/brier_reward": 0.7789061665534973,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.6071614623069763,
|
|
"step": 54
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00455729166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3925.0,
|
|
"completions/mean_length": 851.8372802734375,
|
|
"completions/mean_terminated_length": 836.9849243164062,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"epoch": 0.352,
|
|
"grad_norm": 0.027183230966329575,
|
|
"learning_rate": 1.984126984126984e-06,
|
|
"loss": 0.0057,
|
|
"num_tokens": 87022050.0,
|
|
"reward": 1.2177274227142334,
|
|
"reward_std": 0.1196865662932396,
|
|
"rewards/accuracy_reward": 0.6686198115348816,
|
|
"rewards/brier_reward": 0.7720312476158142,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9947916865348816,
|
|
"rewards/mean_confidence_reward": 0.6061198115348816,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3183.0,
|
|
"completions/mean_length": 914.0345458984375,
|
|
"completions/mean_terminated_length": 901.5562133789062,
|
|
"completions/min_length": 289.0,
|
|
"completions/min_terminated_length": 289.0,
|
|
"epoch": 0.3584,
|
|
"grad_norm": 0.03011305257678032,
|
|
"learning_rate": 1.904761904761905e-06,
|
|
"loss": 0.003,
|
|
"num_tokens": 88961935.0,
|
|
"reward": 1.221256136894226,
|
|
"reward_std": 0.14604595303535461,
|
|
"rewards/accuracy_reward": 0.6731770634651184,
|
|
"rewards/brier_reward": 0.7732291221618652,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.6072916984558105,
|
|
"step": 56
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3942.0,
|
|
"completions/mean_length": 844.2005615234375,
|
|
"completions/mean_terminated_length": 831.4483642578125,
|
|
"completions/min_length": 272.0,
|
|
"completions/min_terminated_length": 272.0,
|
|
"epoch": 0.3648,
|
|
"grad_norm": 0.03051913157105446,
|
|
"learning_rate": 1.8253968253968254e-06,
|
|
"loss": 0.0057,
|
|
"num_tokens": 90792635.0,
|
|
"reward": 1.253857135772705,
|
|
"reward_std": 0.1404409408569336,
|
|
"rewards/accuracy_reward": 0.7265625,
|
|
"rewards/brier_reward": 0.7850456237792969,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.6100911498069763,
|
|
"step": 57
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00846354166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3134.0,
|
|
"completions/mean_length": 901.140625,
|
|
"completions/mean_terminated_length": 873.8699951171875,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"epoch": 0.3712,
|
|
"grad_norm": 0.0313086099922657,
|
|
"learning_rate": 1.746031746031746e-06,
|
|
"loss": 0.0134,
|
|
"num_tokens": 92713755.0,
|
|
"reward": 1.249159812927246,
|
|
"reward_std": 0.1471250057220459,
|
|
"rewards/accuracy_reward": 0.7239583134651184,
|
|
"rewards/brier_reward": 0.7828125357627869,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9915364384651184,
|
|
"rewards/mean_confidence_reward": 0.6104167103767395,
|
|
"step": 58
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.001953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3972.0,
|
|
"completions/mean_length": 883.6341552734375,
|
|
"completions/mean_terminated_length": 877.3477172851562,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"epoch": 0.3776,
|
|
"grad_norm": 0.029742760583758354,
|
|
"learning_rate": 1.6666666666666667e-06,
|
|
"loss": 0.0036,
|
|
"num_tokens": 94602753.0,
|
|
"reward": 1.2164547443389893,
|
|
"reward_std": 0.1401282548904419,
|
|
"rewards/accuracy_reward": 0.6647135615348816,
|
|
"rewards/brier_reward": 0.7714387774467468,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.615039050579071,
|
|
"step": 59
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00716145833333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2962.0,
|
|
"completions/mean_length": 910.7435302734375,
|
|
"completions/mean_terminated_length": 887.7678833007812,
|
|
"completions/min_length": 322.0,
|
|
"completions/min_terminated_length": 322.0,
|
|
"epoch": 0.384,
|
|
"grad_norm": 0.031418535858392715,
|
|
"learning_rate": 1.5873015873015873e-06,
|
|
"loss": 0.0081,
|
|
"num_tokens": 96536247.0,
|
|
"reward": 1.2046709060668945,
|
|
"reward_std": 0.14322175085544586,
|
|
"rewards/accuracy_reward": 0.6510416865348816,
|
|
"rewards/brier_reward": 0.7654492259025574,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9928385615348816,
|
|
"rewards/mean_confidence_reward": 0.6162760853767395,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.384,
|
|
"eval_completions/clipped_ratio": 0.005859375,
|
|
"eval_completions/max_length": 2913.875,
|
|
"eval_completions/max_terminated_length": 2627.125,
|
|
"eval_completions/mean_length": 932.2150726318359,
|
|
"eval_completions/mean_terminated_length": 913.7788391113281,
|
|
"eval_completions/min_length": 368.375,
|
|
"eval_completions/min_terminated_length": 368.375,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 96536247.0,
|
|
"eval_reward": 1.2079066336154938,
|
|
"eval_reward_std": 0.2947797551751137,
|
|
"eval_rewards/accuracy_reward": 0.65625,
|
|
"eval_rewards/brier_reward": 0.7673632651567459,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 0.9921875,
|
|
"eval_rewards/mean_confidence_reward": 0.6195312589406967,
|
|
"eval_runtime": 201.5626,
|
|
"eval_samples_per_second": 4.961,
|
|
"eval_steps_per_second": 0.04,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00455729166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3741.0,
|
|
"completions/mean_length": 987.9108276367188,
|
|
"completions/mean_terminated_length": 973.6814575195312,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"epoch": 0.3904,
|
|
"grad_norm": 0.027266908437013626,
|
|
"learning_rate": 1.507936507936508e-06,
|
|
"loss": 0.0044,
|
|
"num_tokens": 98597286.0,
|
|
"reward": 1.1895896196365356,
|
|
"reward_std": 0.13476239144802094,
|
|
"rewards/accuracy_reward": 0.625,
|
|
"rewards/brier_reward": 0.7593750357627869,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9947916865348816,
|
|
"rewards/mean_confidence_reward": 0.6208333373069763,
|
|
"step": 61
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0032552083333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3804.0,
|
|
"completions/mean_length": 952.4329833984375,
|
|
"completions/mean_terminated_length": 942.16650390625,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"epoch": 0.3968,
|
|
"grad_norm": 0.02792213298380375,
|
|
"learning_rate": 1.4285714285714286e-06,
|
|
"loss": 0.002,
|
|
"num_tokens": 100595607.0,
|
|
"reward": 1.2164483070373535,
|
|
"reward_std": 0.14332908391952515,
|
|
"rewards/accuracy_reward": 0.6634114384651184,
|
|
"rewards/brier_reward": 0.7727277874946594,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.6254557967185974,
|
|
"step": 62
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0026041666666666297,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3669.0,
|
|
"completions/mean_length": 924.58203125,
|
|
"completions/mean_terminated_length": 916.3015747070312,
|
|
"completions/min_length": 274.0,
|
|
"completions/min_terminated_length": 274.0,
|
|
"epoch": 0.4032,
|
|
"grad_norm": 0.033522896468639374,
|
|
"learning_rate": 1.3492063492063493e-06,
|
|
"loss": 0.0054,
|
|
"num_tokens": 102549109.0,
|
|
"reward": 1.2374248504638672,
|
|
"reward_std": 0.1291639655828476,
|
|
"rewards/accuracy_reward": 0.6966145634651184,
|
|
"rewards/brier_reward": 0.7814778685569763,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.627148449420929,
|
|
"step": 63
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00520833333333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3990.0,
|
|
"completions/mean_length": 965.7318115234375,
|
|
"completions/mean_terminated_length": 949.3429565429688,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"epoch": 0.4096,
|
|
"grad_norm": 0.02902921475470066,
|
|
"learning_rate": 1.26984126984127e-06,
|
|
"loss": 0.0089,
|
|
"num_tokens": 104564633.0,
|
|
"reward": 1.2365102767944336,
|
|
"reward_std": 0.14597181975841522,
|
|
"rewards/accuracy_reward": 0.6998698115348816,
|
|
"rewards/brier_reward": 0.7789974212646484,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.994140625,
|
|
"rewards/mean_confidence_reward": 0.6319661140441895,
|
|
"step": 64
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3715.0,
|
|
"completions/mean_length": 950.521484375,
|
|
"completions/mean_terminated_length": 938.186279296875,
|
|
"completions/min_length": 280.0,
|
|
"completions/min_terminated_length": 280.0,
|
|
"epoch": 0.416,
|
|
"grad_norm": 0.027246825397014618,
|
|
"learning_rate": 1.1904761904761906e-06,
|
|
"loss": 0.0039,
|
|
"num_tokens": 106563458.0,
|
|
"reward": 1.2439451217651367,
|
|
"reward_std": 0.13585898280143738,
|
|
"rewards/accuracy_reward": 0.7057291865348816,
|
|
"rewards/brier_reward": 0.7860546112060547,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.634960949420929,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3899.0,
|
|
"completions/mean_length": 952.9557495117188,
|
|
"completions/mean_terminated_length": 948.8578491210938,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"epoch": 0.4224,
|
|
"grad_norm": 0.028497766703367233,
|
|
"learning_rate": 1.111111111111111e-06,
|
|
"loss": 0.0032,
|
|
"num_tokens": 108560534.0,
|
|
"reward": 1.2353613376617432,
|
|
"reward_std": 0.14838117361068726,
|
|
"rewards/accuracy_reward": 0.6907551884651184,
|
|
"rewards/brier_reward": 0.7825586199760437,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9973958134651184,
|
|
"rewards/mean_confidence_reward": 0.6427083611488342,
|
|
"step": 66
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0078125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3713.0,
|
|
"completions/mean_length": 972.2044677734375,
|
|
"completions/mean_terminated_length": 947.6076049804688,
|
|
"completions/min_length": 363.0,
|
|
"completions/min_terminated_length": 363.0,
|
|
"epoch": 0.4288,
|
|
"grad_norm": 0.03186187148094177,
|
|
"learning_rate": 1.0317460317460317e-06,
|
|
"loss": 0.0092,
|
|
"num_tokens": 110585184.0,
|
|
"reward": 1.2379752397537231,
|
|
"reward_std": 0.15858837962150574,
|
|
"rewards/accuracy_reward": 0.7044270634651184,
|
|
"rewards/brier_reward": 0.7799739837646484,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9915364384651184,
|
|
"rewards/mean_confidence_reward": 0.6419270634651184,
|
|
"step": 67
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00520833333333337,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3369.0,
|
|
"completions/mean_length": 955.3737182617188,
|
|
"completions/mean_terminated_length": 938.9306640625,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"epoch": 0.4352,
|
|
"grad_norm": 0.029132414609193802,
|
|
"learning_rate": 9.523809523809525e-07,
|
|
"loss": 0.0043,
|
|
"num_tokens": 112588822.0,
|
|
"reward": 1.2233562469482422,
|
|
"reward_std": 0.14732417464256287,
|
|
"rewards/accuracy_reward": 0.6764323115348816,
|
|
"rewards/brier_reward": 0.775475263595581,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9947916865348816,
|
|
"rewards/mean_confidence_reward": 0.6490885615348816,
|
|
"step": 68
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.001953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3013.0,
|
|
"completions/mean_length": 945.8737182617188,
|
|
"completions/mean_terminated_length": 939.7091064453125,
|
|
"completions/min_length": 259.0,
|
|
"completions/min_terminated_length": 259.0,
|
|
"epoch": 0.4416,
|
|
"grad_norm": 0.03151208162307739,
|
|
"learning_rate": 8.73015873015873e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 114577132.0,
|
|
"reward": 1.20863938331604,
|
|
"reward_std": 0.14611124992370605,
|
|
"rewards/accuracy_reward": 0.650390625,
|
|
"rewards/brier_reward": 0.7688280940055847,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.998046875,
|
|
"rewards/mean_confidence_reward": 0.6532552242279053,
|
|
"step": 69
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 4041.0,
|
|
"completions/max_terminated_length": 4041.0,
|
|
"completions/mean_length": 954.4154052734375,
|
|
"completions/mean_terminated_length": 954.4154052734375,
|
|
"completions/min_length": 305.0,
|
|
"completions/min_terminated_length": 305.0,
|
|
"epoch": 0.448,
|
|
"grad_norm": 0.030807675793766975,
|
|
"learning_rate": 7.936507936507937e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 116578794.0,
|
|
"reward": 1.2335875034332275,
|
|
"reward_std": 0.15881499648094177,
|
|
"rewards/accuracy_reward": 0.685546875,
|
|
"rewards/brier_reward": 0.7822656631469727,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.659375011920929,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0032552083333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3965.0,
|
|
"completions/mean_length": 953.80859375,
|
|
"completions/mean_terminated_length": 943.5466918945312,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"epoch": 0.4544,
|
|
"grad_norm": 0.03015553206205368,
|
|
"learning_rate": 7.142857142857143e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 118573732.0,
|
|
"reward": 1.2622398138046265,
|
|
"reward_std": 0.12950393557548523,
|
|
"rewards/accuracy_reward": 0.734375,
|
|
"rewards/brier_reward": 0.7933464050292969,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9967448115348816,
|
|
"rewards/mean_confidence_reward": 0.6619791388511658,
|
|
"step": 71
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00455729166666663,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 2912.0,
|
|
"completions/mean_length": 944.9674682617188,
|
|
"completions/mean_terminated_length": 930.54150390625,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"epoch": 0.4608,
|
|
"grad_norm": 0.033751603215932846,
|
|
"learning_rate": 6.34920634920635e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 120561458.0,
|
|
"reward": 1.204261302947998,
|
|
"reward_std": 0.16428357362747192,
|
|
"rewards/accuracy_reward": 0.6458333134651184,
|
|
"rewards/brier_reward": 0.7672330737113953,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9954426884651184,
|
|
"rewards/mean_confidence_reward": 0.6617838740348816,
|
|
"step": 72
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4048.0,
|
|
"completions/mean_length": 956.7122802734375,
|
|
"completions/mean_terminated_length": 944.4013061523438,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"epoch": 0.4672,
|
|
"grad_norm": 0.02828790619969368,
|
|
"learning_rate": 5.555555555555555e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 122557752.0,
|
|
"reward": 1.2246062755584717,
|
|
"reward_std": 0.14193803071975708,
|
|
"rewards/accuracy_reward": 0.6751301884651184,
|
|
"rewards/brier_reward": 0.7786262631416321,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9954426884651184,
|
|
"rewards/mean_confidence_reward": 0.6623697876930237,
|
|
"step": 73
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.001953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3056.0,
|
|
"completions/mean_length": 903.2955932617188,
|
|
"completions/mean_terminated_length": 897.0476684570312,
|
|
"completions/min_length": 336.0,
|
|
"completions/min_terminated_length": 336.0,
|
|
"epoch": 0.4736,
|
|
"grad_norm": 0.02815091237425804,
|
|
"learning_rate": 4.7619047619047623e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 124476438.0,
|
|
"reward": 1.2407326698303223,
|
|
"reward_std": 0.12231898307800293,
|
|
"rewards/accuracy_reward": 0.697265625,
|
|
"rewards/brier_reward": 0.7861393094062805,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.998046875,
|
|
"rewards/mean_confidence_reward": 0.668164074420929,
|
|
"step": 74
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.005859375,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3090.0,
|
|
"completions/mean_length": 964.4909057617188,
|
|
"completions/mean_terminated_length": 946.0340576171875,
|
|
"completions/min_length": 385.0,
|
|
"completions/min_terminated_length": 385.0,
|
|
"epoch": 0.48,
|
|
"grad_norm": 0.03320184722542763,
|
|
"learning_rate": 3.9682539682539683e-07,
|
|
"loss": 0.0072,
|
|
"num_tokens": 126488056.0,
|
|
"reward": 1.2050946950912476,
|
|
"reward_std": 0.1566193401813507,
|
|
"rewards/accuracy_reward": 0.6490885615348816,
|
|
"rewards/brier_reward": 0.7675976753234863,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9934895634651184,
|
|
"rewards/mean_confidence_reward": 0.669921875,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.48,
|
|
"eval_completions/clipped_ratio": 0.001953125,
|
|
"eval_completions/max_length": 2631.5,
|
|
"eval_completions/max_terminated_length": 2502.0,
|
|
"eval_completions/mean_length": 937.1688766479492,
|
|
"eval_completions/mean_terminated_length": 931.0681457519531,
|
|
"eval_completions/min_length": 372.375,
|
|
"eval_completions/min_terminated_length": 372.375,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 126488056.0,
|
|
"eval_reward": 1.2203388065099716,
|
|
"eval_reward_std": 0.3088080957531929,
|
|
"eval_rewards/accuracy_reward": 0.66796875,
|
|
"eval_rewards/brier_reward": 0.774648442864418,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 0.998046875,
|
|
"eval_rewards/mean_confidence_reward": 0.6734374910593033,
|
|
"eval_runtime": 181.3836,
|
|
"eval_samples_per_second": 5.513,
|
|
"eval_steps_per_second": 0.044,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.001953125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3846.0,
|
|
"completions/mean_length": 944.2220458984375,
|
|
"completions/mean_terminated_length": 938.0541381835938,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"epoch": 0.4864,
|
|
"grad_norm": 0.03037758357822895,
|
|
"learning_rate": 3.174603174603175e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 128468277.0,
|
|
"reward": 1.2655926942825317,
|
|
"reward_std": 0.14228761196136475,
|
|
"rewards/accuracy_reward": 0.7350260615348816,
|
|
"rewards/brier_reward": 0.7987499237060547,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9973958134651184,
|
|
"rewards/mean_confidence_reward": 0.6697916984558105,
|
|
"step": 76
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00390625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3175.0,
|
|
"completions/mean_length": 930.3236083984375,
|
|
"completions/mean_terminated_length": 917.9091796875,
|
|
"completions/min_length": 305.0,
|
|
"completions/min_terminated_length": 305.0,
|
|
"epoch": 0.4928,
|
|
"grad_norm": 0.030980372801423073,
|
|
"learning_rate": 2.3809523809523811e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 130428270.0,
|
|
"reward": 1.256566047668457,
|
|
"reward_std": 0.137380450963974,
|
|
"rewards/accuracy_reward": 0.724609375,
|
|
"rewards/brier_reward": 0.7930664420127869,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9954426884651184,
|
|
"rewards/mean_confidence_reward": 0.6725910305976868,
|
|
"step": 77
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3606.0,
|
|
"completions/mean_length": 963.6217651367188,
|
|
"completions/mean_terminated_length": 959.5377807617188,
|
|
"completions/min_length": 306.0,
|
|
"completions/min_terminated_length": 306.0,
|
|
"epoch": 0.4992,
|
|
"grad_norm": 0.031181806698441505,
|
|
"learning_rate": 1.5873015873015874e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 132444513.0,
|
|
"reward": 1.2151597738265991,
|
|
"reward_std": 0.15682736039161682,
|
|
"rewards/accuracy_reward": 0.6588541865348816,
|
|
"rewards/brier_reward": 0.7727539539337158,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9986979365348816,
|
|
"rewards/mean_confidence_reward": 0.672070324420929,
|
|
"step": 78
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0026041666666666297,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3335.0,
|
|
"completions/mean_length": 978.9622802734375,
|
|
"completions/mean_terminated_length": 970.8237915039062,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"epoch": 0.5056,
|
|
"grad_norm": 0.030967287719249725,
|
|
"learning_rate": 7.936507936507937e-08,
|
|
"loss": 0.0037,
|
|
"num_tokens": 134483575.0,
|
|
"reward": 1.2452900409698486,
|
|
"reward_std": 0.14523382484912872,
|
|
"rewards/accuracy_reward": 0.7057291865348816,
|
|
"rewards/brier_reward": 0.7874414324760437,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9973958134651184,
|
|
"rewards/mean_confidence_reward": 0.6742838025093079,
|
|
"step": 79
|
|
},
|
|
{
|
|
"epoch": 0.5056,
|
|
"step": 79,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.008544975337631341,
|
|
"train_runtime": 17366.842,
|
|
"train_samples_per_second": 0.864,
|
|
"train_steps_per_second": 0.005
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 79,
|
|
"num_input_tokens_seen": 134483575,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 60,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 3,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|