2292 lines
85 KiB
JSON
2292 lines
85 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.5056,
|
|
"eval_steps": 15,
|
|
"global_step": 79,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0026041666666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3027.0,
|
|
"completions/mean_length": 505.20184326171875,
|
|
"completions/mean_terminated_length": 498.5,
|
|
"completions/min_length": 115.0,
|
|
"completions/min_terminated_length": 115.0,
|
|
"epoch": 0.0064,
|
|
"grad_norm": 0.13465720415115356,
|
|
"learning_rate": 0.0,
|
|
"loss": 0.0149,
|
|
"num_tokens": 1305438.0,
|
|
"reward": 0.17192092537879944,
|
|
"reward_std": 0.2633022665977478,
|
|
"rewards/accuracy_reward": 0.0572916679084301,
|
|
"rewards/brier_reward": 0.05020228028297424,
|
|
"rewards/confidence_one_or_zero": 0.15625,
|
|
"rewards/format_reward": 0.236328125,
|
|
"rewards/mean_confidence_reward": 0.8309497833251953,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2671.0,
|
|
"completions/max_terminated_length": 2671.0,
|
|
"completions/mean_length": 521.638671875,
|
|
"completions/mean_terminated_length": 521.638671875,
|
|
"completions/min_length": 101.0,
|
|
"completions/min_terminated_length": 101.0,
|
|
"epoch": 0.0128,
|
|
"grad_norm": 0.10315462946891785,
|
|
"learning_rate": 3.125e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 2637843.0,
|
|
"reward": 0.1674187034368515,
|
|
"reward_std": 0.21032021939754486,
|
|
"rewards/accuracy_reward": 0.0546875,
|
|
"rewards/brier_reward": 0.047056894749403,
|
|
"rewards/confidence_one_or_zero": 0.1705729216337204,
|
|
"rewards/format_reward": 0.2330729216337204,
|
|
"rewards/mean_confidence_reward": 0.832894504070282,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2148.0,
|
|
"completions/mean_length": 533.7213745117188,
|
|
"completions/mean_terminated_length": 530.4119873046875,
|
|
"completions/min_length": 107.0,
|
|
"completions/min_terminated_length": 107.0,
|
|
"epoch": 0.0192,
|
|
"grad_norm": 0.127846822142601,
|
|
"learning_rate": 6.25e-07,
|
|
"loss": 0.0113,
|
|
"num_tokens": 3991751.0,
|
|
"reward": 0.1596042513847351,
|
|
"reward_std": 0.23278622329235077,
|
|
"rewards/accuracy_reward": 0.048828125,
|
|
"rewards/brier_reward": 0.041193824261426926,
|
|
"rewards/confidence_one_or_zero": 0.1640625,
|
|
"rewards/format_reward": 0.2291666716337204,
|
|
"rewards/mean_confidence_reward": 0.8296234011650085,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2377.0,
|
|
"completions/mean_length": 471.189453125,
|
|
"completions/mean_terminated_length": 467.7985534667969,
|
|
"completions/min_length": 110.0,
|
|
"completions/min_terminated_length": 110.0,
|
|
"epoch": 0.0256,
|
|
"grad_norm": 0.1541290283203125,
|
|
"learning_rate": 9.375000000000001e-07,
|
|
"loss": 0.0162,
|
|
"num_tokens": 5232890.0,
|
|
"reward": 0.18936896324157715,
|
|
"reward_std": 0.2646319568157196,
|
|
"rewards/accuracy_reward": 0.052734375,
|
|
"rewards/brier_reward": 0.039525315165519714,
|
|
"rewards/confidence_one_or_zero": 0.1477864533662796,
|
|
"rewards/format_reward": 0.2864583432674408,
|
|
"rewards/mean_confidence_reward": 0.8490968346595764,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2192.0,
|
|
"completions/mean_length": 485.79949951171875,
|
|
"completions/mean_terminated_length": 482.4276428222656,
|
|
"completions/min_length": 129.0,
|
|
"completions/min_terminated_length": 129.0,
|
|
"epoch": 0.032,
|
|
"grad_norm": 0.1194499284029007,
|
|
"learning_rate": 1.25e-06,
|
|
"loss": 0.0171,
|
|
"num_tokens": 6507478.0,
|
|
"reward": 0.19178208708763123,
|
|
"reward_std": 0.24853277206420898,
|
|
"rewards/accuracy_reward": 0.041015625,
|
|
"rewards/brier_reward": 0.030679741874337196,
|
|
"rewards/confidence_one_or_zero": 0.1490885466337204,
|
|
"rewards/format_reward": 0.3118489682674408,
|
|
"rewards/mean_confidence_reward": 0.8418641090393066,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.001953125,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2024.0,
|
|
"completions/mean_length": 459.5013122558594,
|
|
"completions/mean_terminated_length": 454.3887939453125,
|
|
"completions/min_length": 109.0,
|
|
"completions/min_terminated_length": 109.0,
|
|
"epoch": 0.0384,
|
|
"grad_norm": 0.126247376203537,
|
|
"learning_rate": 1.5625e-06,
|
|
"loss": 0.0242,
|
|
"num_tokens": 7743552.0,
|
|
"reward": 0.33419346809387207,
|
|
"reward_std": 0.3138011693954468,
|
|
"rewards/accuracy_reward": 0.095703125,
|
|
"rewards/brier_reward": 0.08373213559389114,
|
|
"rewards/confidence_one_or_zero": 0.0846354141831398,
|
|
"rewards/format_reward": 0.4889322817325592,
|
|
"rewards/mean_confidence_reward": 0.8835051655769348,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2354.0,
|
|
"completions/mean_length": 394.1946716308594,
|
|
"completions/mean_terminated_length": 392.4501647949219,
|
|
"completions/min_length": 112.0,
|
|
"completions/min_terminated_length": 112.0,
|
|
"epoch": 0.0448,
|
|
"grad_norm": 0.10085690021514893,
|
|
"learning_rate": 1.8750000000000003e-06,
|
|
"loss": 0.0159,
|
|
"num_tokens": 8880763.0,
|
|
"reward": 0.5068206787109375,
|
|
"reward_std": 0.2961437702178955,
|
|
"rewards/accuracy_reward": 0.126953125,
|
|
"rewards/brier_reward": 0.11583596467971802,
|
|
"rewards/confidence_one_or_zero": 0.0397135429084301,
|
|
"rewards/format_reward": 0.7708333134651184,
|
|
"rewards/mean_confidence_reward": 0.9076032042503357,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2949.0,
|
|
"completions/mean_length": 358.37109375,
|
|
"completions/mean_terminated_length": 356.6032409667969,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"epoch": 0.0512,
|
|
"grad_norm": 0.0991978868842125,
|
|
"learning_rate": 2.1875000000000002e-06,
|
|
"loss": 0.0158,
|
|
"num_tokens": 9955733.0,
|
|
"reward": 0.5604633092880249,
|
|
"reward_std": 0.2917187809944153,
|
|
"rewards/accuracy_reward": 0.1399739533662796,
|
|
"rewards/brier_reward": 0.1339281052350998,
|
|
"rewards/confidence_one_or_zero": 0.0494791679084301,
|
|
"rewards/format_reward": 0.8470051884651184,
|
|
"rewards/mean_confidence_reward": 0.9129166603088379,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1473.0,
|
|
"completions/max_terminated_length": 1473.0,
|
|
"completions/mean_length": 307.18426513671875,
|
|
"completions/mean_terminated_length": 307.18426513671875,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"epoch": 0.0576,
|
|
"grad_norm": 0.09707837551832199,
|
|
"learning_rate": 2.5e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 10960744.0,
|
|
"reward": 0.628527045249939,
|
|
"reward_std": 0.26747146248817444,
|
|
"rewards/accuracy_reward": 0.150390625,
|
|
"rewards/brier_reward": 0.16002988815307617,
|
|
"rewards/confidence_one_or_zero": 0.02799479104578495,
|
|
"rewards/format_reward": 0.9466145634651184,
|
|
"rewards/mean_confidence_reward": 0.9208442568778992,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 910.0,
|
|
"completions/max_terminated_length": 910.0,
|
|
"completions/mean_length": 269.494140625,
|
|
"completions/mean_terminated_length": 269.494140625,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"epoch": 0.064,
|
|
"grad_norm": 0.11624328792095184,
|
|
"learning_rate": 2.8125e-06,
|
|
"loss": 0.0038,
|
|
"num_tokens": 11911095.0,
|
|
"reward": 0.6837179064750671,
|
|
"reward_std": 0.27561697363853455,
|
|
"rewards/accuracy_reward": 0.1959635466337204,
|
|
"rewards/brier_reward": 0.2241881638765335,
|
|
"rewards/confidence_one_or_zero": 0.02083333395421505,
|
|
"rewards/format_reward": 0.947265625,
|
|
"rewards/mean_confidence_reward": 0.90611332654953,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1604.0,
|
|
"completions/max_terminated_length": 1604.0,
|
|
"completions/mean_length": 262.5677185058594,
|
|
"completions/mean_terminated_length": 262.5677185058594,
|
|
"completions/min_length": 99.0,
|
|
"completions/min_terminated_length": 99.0,
|
|
"epoch": 0.0704,
|
|
"grad_norm": 0.10577341914176941,
|
|
"learning_rate": 3.125e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 12842727.0,
|
|
"reward": 0.6731219291687012,
|
|
"reward_std": 0.22560656070709229,
|
|
"rewards/accuracy_reward": 0.1490885466337204,
|
|
"rewards/brier_reward": 0.22317902743816376,
|
|
"rewards/confidence_one_or_zero": 0.009114583022892475,
|
|
"rewards/format_reward": 0.9739583134651184,
|
|
"rewards/mean_confidence_reward": 0.8843294978141785,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1646.0,
|
|
"completions/max_terminated_length": 1646.0,
|
|
"completions/mean_length": 236.234375,
|
|
"completions/mean_terminated_length": 236.234375,
|
|
"completions/min_length": 80.0,
|
|
"completions/min_terminated_length": 80.0,
|
|
"epoch": 0.0768,
|
|
"grad_norm": 0.09674405306577682,
|
|
"learning_rate": 3.4375e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 13728367.0,
|
|
"reward": 0.6834604740142822,
|
|
"reward_std": 0.19712857902050018,
|
|
"rewards/accuracy_reward": 0.1263020783662796,
|
|
"rewards/brier_reward": 0.247111976146698,
|
|
"rewards/confidence_one_or_zero": 0.005859375,
|
|
"rewards/format_reward": 0.9934895634651184,
|
|
"rewards/mean_confidence_reward": 0.85888671875,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1216.0,
|
|
"completions/max_terminated_length": 1216.0,
|
|
"completions/mean_length": 211.8619842529297,
|
|
"completions/mean_terminated_length": 211.8619842529297,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"epoch": 0.0832,
|
|
"grad_norm": 0.1306491196155548,
|
|
"learning_rate": 3.7500000000000005e-06,
|
|
"loss": -0.0014,
|
|
"num_tokens": 14579315.0,
|
|
"reward": 0.779711902141571,
|
|
"reward_std": 0.2249181866645813,
|
|
"rewards/accuracy_reward": 0.1608072966337204,
|
|
"rewards/brier_reward": 0.4103199541568756,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.98828125,
|
|
"rewards/mean_confidence_reward": 0.7606966495513916,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 477.0,
|
|
"completions/mean_length": 180.619140625,
|
|
"completions/mean_terminated_length": 178.73550415039062,
|
|
"completions/min_length": 82.0,
|
|
"completions/min_terminated_length": 82.0,
|
|
"epoch": 0.0896,
|
|
"grad_norm": 0.0933312326669693,
|
|
"learning_rate": 4.0625000000000005e-06,
|
|
"loss": -0.0018,
|
|
"num_tokens": 15384058.0,
|
|
"reward": 0.8816039562225342,
|
|
"reward_std": 0.17225664854049683,
|
|
"rewards/accuracy_reward": 0.2044270783662796,
|
|
"rewards/brier_reward": 0.559418797492981,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.6580598950386047,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 885.0,
|
|
"completions/max_terminated_length": 885.0,
|
|
"completions/mean_length": 171.978515625,
|
|
"completions/mean_terminated_length": 171.978515625,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"epoch": 0.096,
|
|
"grad_norm": 0.08225424587726593,
|
|
"learning_rate": 4.3750000000000005e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 16183017.0,
|
|
"reward": 0.8857101798057556,
|
|
"reward_std": 0.1458606719970703,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/brier_reward": 0.6047414541244507,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.6036458611488342,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.096,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 477.625,
|
|
"eval_completions/max_terminated_length": 477.625,
|
|
"eval_completions/mean_length": 165.50653648376465,
|
|
"eval_completions/mean_terminated_length": 165.50653648376465,
|
|
"eval_completions/min_length": 88.375,
|
|
"eval_completions/min_terminated_length": 88.375,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 16183017.0,
|
|
"eval_reward": 0.9013179466128349,
|
|
"eval_reward_std": 0.20355869084596634,
|
|
"eval_rewards/accuracy_reward": 0.158203125,
|
|
"eval_rewards/brier_reward": 0.6463748663663864,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 0.998046875,
|
|
"eval_rewards/mean_confidence_reward": 0.5507324263453484,
|
|
"eval_runtime": 45.1835,
|
|
"eval_samples_per_second": 22.132,
|
|
"eval_steps_per_second": 0.177,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 614.0,
|
|
"completions/max_terminated_length": 614.0,
|
|
"completions/mean_length": 166.12435913085938,
|
|
"completions/mean_terminated_length": 166.12435913085938,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"epoch": 0.1024,
|
|
"grad_norm": 0.08497676253318787,
|
|
"learning_rate": 4.6875000000000004e-06,
|
|
"loss": -0.0014,
|
|
"num_tokens": 16972960.0,
|
|
"reward": 0.9230477213859558,
|
|
"reward_std": 0.1452154517173767,
|
|
"rewards/accuracy_reward": 0.19140625,
|
|
"rewards/brier_reward": 0.655329167842865,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.5518229603767395,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2449.0,
|
|
"completions/max_terminated_length": 2449.0,
|
|
"completions/mean_length": 165.8404998779297,
|
|
"completions/mean_terminated_length": 165.8404998779297,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"epoch": 0.1088,
|
|
"grad_norm": 0.06753753870725632,
|
|
"learning_rate": 5e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 17759891.0,
|
|
"reward": 0.9658437967300415,
|
|
"reward_std": 0.10605766624212265,
|
|
"rewards/accuracy_reward": 0.154296875,
|
|
"rewards/brier_reward": 0.7780342102050781,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.3663736879825592,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 445.0,
|
|
"completions/max_terminated_length": 445.0,
|
|
"completions/mean_length": 161.77670288085938,
|
|
"completions/mean_terminated_length": 161.77670288085938,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"epoch": 0.1152,
|
|
"grad_norm": 0.031030012294650078,
|
|
"learning_rate": 4.920634920634921e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 18531852.0,
|
|
"reward": 0.9964861869812012,
|
|
"reward_std": 0.049115654081106186,
|
|
"rewards/accuracy_reward": 0.1321614533662796,
|
|
"rewards/brier_reward": 0.8608064651489258,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.2228190153837204,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 860.0,
|
|
"completions/max_terminated_length": 860.0,
|
|
"completions/mean_length": 158.921875,
|
|
"completions/mean_terminated_length": 158.921875,
|
|
"completions/min_length": 76.0,
|
|
"completions/min_terminated_length": 76.0,
|
|
"epoch": 0.1216,
|
|
"grad_norm": 0.027188602834939957,
|
|
"learning_rate": 4.841269841269842e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 19304708.0,
|
|
"reward": 1.004211664199829,
|
|
"reward_std": 0.04009218513965607,
|
|
"rewards/accuracy_reward": 0.15234375,
|
|
"rewards/brier_reward": 0.856076180934906,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.1702994853258133,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 482.0,
|
|
"completions/max_terminated_length": 482.0,
|
|
"completions/mean_length": 156.421875,
|
|
"completions/mean_terminated_length": 156.421875,
|
|
"completions/min_length": 75.0,
|
|
"completions/min_terminated_length": 75.0,
|
|
"epoch": 0.128,
|
|
"grad_norm": 0.08922947198152542,
|
|
"learning_rate": 4.761904761904762e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 20070732.0,
|
|
"reward": 0.997628390789032,
|
|
"reward_std": 0.03681759163737297,
|
|
"rewards/accuracy_reward": 0.1236979141831398,
|
|
"rewards/brier_reward": 0.8780671954154968,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9934895634651184,
|
|
"rewards/mean_confidence_reward": 0.108072929084301,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 596.0,
|
|
"completions/max_terminated_length": 596.0,
|
|
"completions/mean_length": 149.6217498779297,
|
|
"completions/mean_terminated_length": 149.6217498779297,
|
|
"completions/min_length": 75.0,
|
|
"completions/min_terminated_length": 75.0,
|
|
"epoch": 0.1344,
|
|
"grad_norm": 0.013877675868570805,
|
|
"learning_rate": 4.682539682539683e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 20829247.0,
|
|
"reward": 1.0045512914657593,
|
|
"reward_std": 0.017113717272877693,
|
|
"rewards/accuracy_reward": 0.1256510466337204,
|
|
"rewards/brier_reward": 0.883449375629425,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.1027018204331398,
|
|
"step": 21
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 750.0,
|
|
"completions/max_terminated_length": 750.0,
|
|
"completions/mean_length": 140.603515625,
|
|
"completions/mean_terminated_length": 140.603515625,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"epoch": 0.1408,
|
|
"grad_norm": 0.016994895413517952,
|
|
"learning_rate": 4.603174603174604e-06,
|
|
"loss": -0.0004,
|
|
"num_tokens": 21569022.0,
|
|
"reward": 0.9996089935302734,
|
|
"reward_std": 0.016458362340927124,
|
|
"rewards/accuracy_reward": 0.08203125,
|
|
"rewards/brier_reward": 0.9178356528282166,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.10055339336395264,
|
|
"step": 22
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 662.0,
|
|
"completions/max_terminated_length": 662.0,
|
|
"completions/mean_length": 143.7447967529297,
|
|
"completions/mean_terminated_length": 143.7447967529297,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"epoch": 0.1472,
|
|
"grad_norm": 0.02281384915113449,
|
|
"learning_rate": 4.523809523809524e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 22324550.0,
|
|
"reward": 1.0032870769500732,
|
|
"reward_std": 0.020587272942066193,
|
|
"rewards/accuracy_reward": 0.1178385391831398,
|
|
"rewards/brier_reward": 0.8893844485282898,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.09973958879709244,
|
|
"step": 23
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 437.0,
|
|
"completions/max_terminated_length": 437.0,
|
|
"completions/mean_length": 140.78125,
|
|
"completions/mean_terminated_length": 140.78125,
|
|
"completions/min_length": 75.0,
|
|
"completions/min_terminated_length": 75.0,
|
|
"epoch": 0.1536,
|
|
"grad_norm": 0.01415838673710823,
|
|
"learning_rate": 4.444444444444444e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 23066590.0,
|
|
"reward": 1.004532814025879,
|
|
"reward_std": 0.01972999982535839,
|
|
"rewards/accuracy_reward": 0.1236979141831398,
|
|
"rewards/brier_reward": 0.8853656649589539,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09827474504709244,
|
|
"step": 24
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 343.0,
|
|
"completions/max_terminated_length": 343.0,
|
|
"completions/mean_length": 137.77279663085938,
|
|
"completions/mean_terminated_length": 137.77279663085938,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"epoch": 0.16,
|
|
"grad_norm": 0.012159720994532108,
|
|
"learning_rate": 4.365079365079366e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 23799953.0,
|
|
"reward": 1.0047153234481812,
|
|
"reward_std": 0.01586366444826126,
|
|
"rewards/accuracy_reward": 0.1243489608168602,
|
|
"rewards/brier_reward": 0.8850796222686768,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09500651806592941,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 385.0,
|
|
"completions/max_terminated_length": 385.0,
|
|
"completions/mean_length": 134.890625,
|
|
"completions/mean_terminated_length": 134.890625,
|
|
"completions/min_length": 70.0,
|
|
"completions/min_terminated_length": 70.0,
|
|
"epoch": 0.1664,
|
|
"grad_norm": 0.012517135590314865,
|
|
"learning_rate": 4.2857142857142855e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 24537297.0,
|
|
"reward": 1.0028703212738037,
|
|
"reward_std": 0.014845854602754116,
|
|
"rewards/accuracy_reward": 0.1048177108168602,
|
|
"rewards/brier_reward": 0.9009209275245667,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.0921224057674408,
|
|
"step": 26
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 359.0,
|
|
"completions/max_terminated_length": 359.0,
|
|
"completions/mean_length": 131.251953125,
|
|
"completions/mean_terminated_length": 131.251953125,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"epoch": 0.1728,
|
|
"grad_norm": 0.013604911044239998,
|
|
"learning_rate": 4.206349206349207e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 25259356.0,
|
|
"reward": 1.0048683881759644,
|
|
"reward_std": 0.01563373953104019,
|
|
"rewards/accuracy_reward": 0.1236979141831398,
|
|
"rewards/brier_reward": 0.8860370516777039,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.08803385496139526,
|
|
"step": 27
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 460.0,
|
|
"completions/max_terminated_length": 460.0,
|
|
"completions/mean_length": 134.3873748779297,
|
|
"completions/mean_terminated_length": 134.3873748779297,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"epoch": 0.1792,
|
|
"grad_norm": 0.01096460409462452,
|
|
"learning_rate": 4.126984126984127e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 25992247.0,
|
|
"reward": 1.0031170845031738,
|
|
"reward_std": 0.013636252842843533,
|
|
"rewards/accuracy_reward": 0.1048177108168602,
|
|
"rewards/brier_reward": 0.901414692401886,
|
|
"rewards/confidence_one_or_zero": 0.0013020833721384406,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.08391926437616348,
|
|
"step": 28
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 362.0,
|
|
"completions/max_terminated_length": 362.0,
|
|
"completions/mean_length": 129.1015625,
|
|
"completions/mean_terminated_length": 129.1015625,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"epoch": 0.1856,
|
|
"grad_norm": 0.01275012455880642,
|
|
"learning_rate": 4.047619047619048e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 26710579.0,
|
|
"reward": 1.0048203468322754,
|
|
"reward_std": 0.014752764254808426,
|
|
"rewards/accuracy_reward": 0.115234375,
|
|
"rewards/brier_reward": 0.8944045901298523,
|
|
"rewards/confidence_one_or_zero": 0.0013020833721384406,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.08635415881872177,
|
|
"step": 29
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 453.0,
|
|
"completions/max_terminated_length": 453.0,
|
|
"completions/mean_length": 130.66732788085938,
|
|
"completions/mean_terminated_length": 130.66732788085938,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.192,
|
|
"grad_norm": 0.013617802411317825,
|
|
"learning_rate": 3.968253968253968e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 27434524.0,
|
|
"reward": 1.0068315267562866,
|
|
"reward_std": 0.016765041276812553,
|
|
"rewards/accuracy_reward": 0.126953125,
|
|
"rewards/brier_reward": 0.8867078423500061,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09343751519918442,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.192,
|
|
"eval_completions/clipped_ratio": 0.0009765625,
|
|
"eval_completions/max_length": 667.0,
|
|
"eval_completions/max_terminated_length": 323.5,
|
|
"eval_completions/mean_length": 133.700345993042,
|
|
"eval_completions/mean_terminated_length": 130.82912826538086,
|
|
"eval_completions/min_length": 76.375,
|
|
"eval_completions/min_terminated_length": 76.375,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 27434524.0,
|
|
"eval_reward": 1.0051013007760048,
|
|
"eval_reward_std": 0.02839039429090917,
|
|
"eval_rewards/accuracy_reward": 0.1181640625,
|
|
"eval_rewards/brier_reward": 0.8930131196975708,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 0.9990234375,
|
|
"eval_rewards/mean_confidence_reward": 0.09717773646116257,
|
|
"eval_runtime": 57.6326,
|
|
"eval_samples_per_second": 17.351,
|
|
"eval_steps_per_second": 0.139,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1659.0,
|
|
"completions/max_terminated_length": 1659.0,
|
|
"completions/mean_length": 132.779296875,
|
|
"completions/mean_terminated_length": 132.779296875,
|
|
"completions/min_length": 73.0,
|
|
"completions/min_terminated_length": 73.0,
|
|
"epoch": 0.1984,
|
|
"grad_norm": 0.012990299612283707,
|
|
"learning_rate": 3.88888888888889e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 28168761.0,
|
|
"reward": 1.0047649145126343,
|
|
"reward_std": 0.014993640594184399,
|
|
"rewards/accuracy_reward": 0.10546875,
|
|
"rewards/brier_reward": 0.9040589332580566,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09805339574813843,
|
|
"step": 31
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 351.0,
|
|
"completions/mean_length": 130.7994842529297,
|
|
"completions/mean_terminated_length": 128.88339233398438,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"epoch": 0.2048,
|
|
"grad_norm": 0.025808464735746384,
|
|
"learning_rate": 3.80952380952381e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 28891869.0,
|
|
"reward": 1.002500295639038,
|
|
"reward_std": 0.019921835511922836,
|
|
"rewards/accuracy_reward": 0.1015625,
|
|
"rewards/brier_reward": 0.9053890109062195,
|
|
"rewards/confidence_one_or_zero": 0.0013020833721384406,
|
|
"rewards/format_reward": 0.998046875,
|
|
"rewards/mean_confidence_reward": 0.09907551854848862,
|
|
"step": 32
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 443.0,
|
|
"completions/max_terminated_length": 443.0,
|
|
"completions/mean_length": 126.14909362792969,
|
|
"completions/mean_terminated_length": 126.14909362792969,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"epoch": 0.2112,
|
|
"grad_norm": 0.014052917249500751,
|
|
"learning_rate": 3.7301587301587305e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 29610298.0,
|
|
"reward": 1.0069693326950073,
|
|
"reward_std": 0.018085956573486328,
|
|
"rewards/accuracy_reward": 0.1263020783662796,
|
|
"rewards/brier_reward": 0.8876343369483948,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09932291507720947,
|
|
"step": 33
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 566.0,
|
|
"completions/max_terminated_length": 566.0,
|
|
"completions/mean_length": 126.43620300292969,
|
|
"completions/mean_terminated_length": 126.43620300292969,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"epoch": 0.2176,
|
|
"grad_norm": 0.018887193873524666,
|
|
"learning_rate": 3.6507936507936507e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 30328656.0,
|
|
"reward": 1.0054115056991577,
|
|
"reward_std": 0.01754041761159897,
|
|
"rewards/accuracy_reward": 0.1178385391831398,
|
|
"rewards/brier_reward": 0.8936333060264587,
|
|
"rewards/confidence_one_or_zero": 0.001953125,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.09931641072034836,
|
|
"step": 34
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 474.0,
|
|
"completions/max_terminated_length": 474.0,
|
|
"completions/mean_length": 124.4765625,
|
|
"completions/mean_terminated_length": 124.4765625,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.224,
|
|
"grad_norm": 0.012697046622633934,
|
|
"learning_rate": 3.5714285714285718e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 31047732.0,
|
|
"reward": 1.0078935623168945,
|
|
"reward_std": 0.014520572498440742,
|
|
"rewards/accuracy_reward": 0.1354166716337204,
|
|
"rewards/brier_reward": 0.8803682923316956,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09940104931592941,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 338.0,
|
|
"completions/max_terminated_length": 338.0,
|
|
"completions/mean_length": 122.73828125,
|
|
"completions/mean_terminated_length": 122.73828125,
|
|
"completions/min_length": 70.0,
|
|
"completions/min_terminated_length": 70.0,
|
|
"epoch": 0.2304,
|
|
"grad_norm": 0.013155413791537285,
|
|
"learning_rate": 3.492063492063492e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 31765642.0,
|
|
"reward": 1.0064831972122192,
|
|
"reward_std": 0.01641521044075489,
|
|
"rewards/accuracy_reward": 0.1223958358168602,
|
|
"rewards/brier_reward": 0.8905684947967529,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09951823204755783,
|
|
"step": 36
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 362.0,
|
|
"completions/max_terminated_length": 362.0,
|
|
"completions/mean_length": 119.89388275146484,
|
|
"completions/mean_terminated_length": 119.89388275146484,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"epoch": 0.2368,
|
|
"grad_norm": 0.01213972270488739,
|
|
"learning_rate": 3.412698412698413e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 32475223.0,
|
|
"reward": 1.0039710998535156,
|
|
"reward_std": 0.013057848438620567,
|
|
"rewards/accuracy_reward": 0.0963541641831398,
|
|
"rewards/brier_reward": 0.9115857481956482,
|
|
"rewards/confidence_one_or_zero": 0.0013020833721384406,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.0997200608253479,
|
|
"step": 37
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 454.0,
|
|
"completions/max_terminated_length": 454.0,
|
|
"completions/mean_length": 118.41536712646484,
|
|
"completions/mean_terminated_length": 118.41536712646484,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"epoch": 0.2432,
|
|
"grad_norm": 0.013623026199638844,
|
|
"learning_rate": 3.3333333333333333e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 33184445.0,
|
|
"reward": 1.0053181648254395,
|
|
"reward_std": 0.014279328286647797,
|
|
"rewards/accuracy_reward": 0.1106770858168602,
|
|
"rewards/brier_reward": 0.899957001209259,
|
|
"rewards/confidence_one_or_zero": 0.0013020833721384406,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09962239861488342,
|
|
"step": 38
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 558.0,
|
|
"completions/max_terminated_length": 558.0,
|
|
"completions/mean_length": 118.62890625,
|
|
"completions/mean_terminated_length": 118.62890625,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"epoch": 0.2496,
|
|
"grad_norm": 0.013208958320319653,
|
|
"learning_rate": 3.2539682539682544e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 33890723.0,
|
|
"reward": 1.0091761350631714,
|
|
"reward_std": 0.01572965644299984,
|
|
"rewards/accuracy_reward": 0.1484375,
|
|
"rewards/brier_reward": 0.8699125647544861,
|
|
"rewards/confidence_one_or_zero": 0.0013020833721384406,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09973958879709244,
|
|
"step": 39
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 392.0,
|
|
"completions/max_terminated_length": 392.0,
|
|
"completions/mean_length": 118.16862487792969,
|
|
"completions/mean_terminated_length": 118.16862487792969,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"epoch": 0.256,
|
|
"grad_norm": 0.013648001477122307,
|
|
"learning_rate": 3.1746031746031746e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 34601606.0,
|
|
"reward": 1.0076011419296265,
|
|
"reward_std": 0.015628747642040253,
|
|
"rewards/accuracy_reward": 0.1328125,
|
|
"rewards/brier_reward": 0.882387638092041,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09990235418081284,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 350.0,
|
|
"completions/max_terminated_length": 350.0,
|
|
"completions/mean_length": 118.84700775146484,
|
|
"completions/mean_terminated_length": 118.84700775146484,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"epoch": 0.2624,
|
|
"grad_norm": 0.011514564044773579,
|
|
"learning_rate": 3.0952380952380957e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 35310251.0,
|
|
"reward": 1.0050663948059082,
|
|
"reward_std": 0.013147103600203991,
|
|
"rewards/accuracy_reward": 0.107421875,
|
|
"rewards/brier_reward": 0.9027087092399597,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09996744990348816,
|
|
"step": 41
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 460.0,
|
|
"completions/max_terminated_length": 460.0,
|
|
"completions/mean_length": 119.25,
|
|
"completions/mean_terminated_length": 119.25,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.2688,
|
|
"grad_norm": 0.019798044115304947,
|
|
"learning_rate": 3.015873015873016e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 36020347.0,
|
|
"reward": 1.0081477165222168,
|
|
"reward_std": 0.01928836479783058,
|
|
"rewards/accuracy_reward": 0.1438802033662796,
|
|
"rewards/brier_reward": 0.8730641007423401,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 42
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 598.0,
|
|
"completions/max_terminated_length": 598.0,
|
|
"completions/mean_length": 118.93229675292969,
|
|
"completions/mean_terminated_length": 118.93229675292969,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"epoch": 0.2752,
|
|
"grad_norm": 0.013396105729043484,
|
|
"learning_rate": 2.936507936507937e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 36730435.0,
|
|
"reward": 1.0055217742919922,
|
|
"reward_std": 0.015360641293227673,
|
|
"rewards/accuracy_reward": 0.1119791641831398,
|
|
"rewards/brier_reward": 0.8990621566772461,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10003254562616348,
|
|
"step": 43
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 367.0,
|
|
"completions/max_terminated_length": 367.0,
|
|
"completions/mean_length": 118.482421875,
|
|
"completions/mean_terminated_length": 118.482421875,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"epoch": 0.2816,
|
|
"grad_norm": 0.014627665281295776,
|
|
"learning_rate": 2.8571428571428573e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 37439320.0,
|
|
"reward": 1.0096123218536377,
|
|
"reward_std": 0.01807526871562004,
|
|
"rewards/accuracy_reward": 0.15234375,
|
|
"rewards/brier_reward": 0.8668785691261292,
|
|
"rewards/confidence_one_or_zero": 0.001953125,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.0997721329331398,
|
|
"step": 44
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 432.0,
|
|
"completions/max_terminated_length": 432.0,
|
|
"completions/mean_length": 120.60091400146484,
|
|
"completions/mean_terminated_length": 120.60091400146484,
|
|
"completions/min_length": 64.0,
|
|
"completions/min_terminated_length": 64.0,
|
|
"epoch": 0.288,
|
|
"grad_norm": 0.01373676024377346,
|
|
"learning_rate": 2.7777777777777783e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 38149899.0,
|
|
"reward": 1.0063700675964355,
|
|
"reward_std": 0.016239743679761887,
|
|
"rewards/accuracy_reward": 0.1204427108168602,
|
|
"rewards/brier_reward": 0.8922954201698303,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10003256052732468,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.288,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 326.375,
|
|
"eval_completions/max_terminated_length": 326.375,
|
|
"eval_completions/mean_length": 121.31820964813232,
|
|
"eval_completions/mean_terminated_length": 121.31820964813232,
|
|
"eval_completions/min_length": 75.375,
|
|
"eval_completions/min_terminated_length": 75.375,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 38149899.0,
|
|
"eval_reward": 1.0062207281589508,
|
|
"eval_reward_std": 0.02544891880825162,
|
|
"eval_rewards/accuracy_reward": 0.119140625,
|
|
"eval_rewards/brier_reward": 0.8932988196611404,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.10019531287252903,
|
|
"eval_runtime": 32.2385,
|
|
"eval_samples_per_second": 31.019,
|
|
"eval_steps_per_second": 0.248,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 465.0,
|
|
"completions/max_terminated_length": 465.0,
|
|
"completions/mean_length": 125.98503112792969,
|
|
"completions/mean_terminated_length": 125.98503112792969,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"epoch": 0.2944,
|
|
"grad_norm": 0.01377463061362505,
|
|
"learning_rate": 2.6984126984126986e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 38875380.0,
|
|
"reward": 1.0058658123016357,
|
|
"reward_std": 0.016262352466583252,
|
|
"rewards/accuracy_reward": 0.115234375,
|
|
"rewards/brier_reward": 0.8964949250221252,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09991536289453506,
|
|
"step": 46
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 413.0,
|
|
"completions/max_terminated_length": 413.0,
|
|
"completions/mean_length": 121.22331237792969,
|
|
"completions/mean_terminated_length": 121.22331237792969,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"epoch": 0.3008,
|
|
"grad_norm": 0.013477513566613197,
|
|
"learning_rate": 2.6190476190476192e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 39589907.0,
|
|
"reward": 1.0082892179489136,
|
|
"reward_std": 0.016982030123472214,
|
|
"rewards/accuracy_reward": 0.1393229216337204,
|
|
"rewards/brier_reward": 0.8772532939910889,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09996744990348816,
|
|
"step": 47
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 596.0,
|
|
"completions/max_terminated_length": 596.0,
|
|
"completions/mean_length": 126.13607025146484,
|
|
"completions/mean_terminated_length": 126.13607025146484,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"epoch": 0.3072,
|
|
"grad_norm": 0.014580963179469109,
|
|
"learning_rate": 2.53968253968254e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 40317932.0,
|
|
"reward": 1.009338617324829,
|
|
"reward_std": 0.0193769708275795,
|
|
"rewards/accuracy_reward": 0.1497395783662796,
|
|
"rewards/brier_reward": 0.8689355850219727,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09996744990348816,
|
|
"step": 48
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 465.0,
|
|
"completions/max_terminated_length": 465.0,
|
|
"completions/mean_length": 124.93034362792969,
|
|
"completions/mean_terminated_length": 124.93034362792969,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"epoch": 0.3136,
|
|
"grad_norm": 0.012050064280629158,
|
|
"learning_rate": 2.4603174603174605e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 41040441.0,
|
|
"reward": 1.0075663328170776,
|
|
"reward_std": 0.014553435146808624,
|
|
"rewards/accuracy_reward": 0.1321614533662796,
|
|
"rewards/brier_reward": 0.882969081401825,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 49
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 683.0,
|
|
"completions/max_terminated_length": 683.0,
|
|
"completions/mean_length": 126.69140625,
|
|
"completions/mean_terminated_length": 126.69140625,
|
|
"completions/min_length": 63.0,
|
|
"completions/min_terminated_length": 63.0,
|
|
"epoch": 0.32,
|
|
"grad_norm": 0.014088026247918606,
|
|
"learning_rate": 2.380952380952381e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 41761151.0,
|
|
"reward": 1.0082541704177856,
|
|
"reward_std": 0.01735132560133934,
|
|
"rewards/accuracy_reward": 0.1393229216337204,
|
|
"rewards/brier_reward": 0.877183198928833,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10019531100988388,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 807.0,
|
|
"completions/max_terminated_length": 807.0,
|
|
"completions/mean_length": 124.443359375,
|
|
"completions/mean_terminated_length": 124.443359375,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"epoch": 0.3264,
|
|
"grad_norm": 0.01315248105674982,
|
|
"learning_rate": 2.301587301587302e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 42476552.0,
|
|
"reward": 1.0089362859725952,
|
|
"reward_std": 0.01715516671538353,
|
|
"rewards/accuracy_reward": 0.1451822966337204,
|
|
"rewards/brier_reward": 0.8726881146430969,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10003254562616348,
|
|
"step": 51
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 427.0,
|
|
"completions/max_terminated_length": 427.0,
|
|
"completions/mean_length": 127.63802337646484,
|
|
"completions/mean_terminated_length": 127.63802337646484,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.3328,
|
|
"grad_norm": 0.013727507553994656,
|
|
"learning_rate": 2.222222222222222e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 43203980.0,
|
|
"reward": 1.0073695182800293,
|
|
"reward_std": 0.015747644007205963,
|
|
"rewards/accuracy_reward": 0.1302083283662796,
|
|
"rewards/brier_reward": 0.884528636932373,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 52
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1262.0,
|
|
"completions/max_terminated_length": 1262.0,
|
|
"completions/mean_length": 130.1881561279297,
|
|
"completions/mean_terminated_length": 130.1881561279297,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"epoch": 0.3392,
|
|
"grad_norm": 0.013043698854744434,
|
|
"learning_rate": 2.1428571428571427e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 43927261.0,
|
|
"reward": 1.007895588874817,
|
|
"reward_std": 0.018076356500387192,
|
|
"rewards/accuracy_reward": 0.1354166716337204,
|
|
"rewards/brier_reward": 0.8803723454475403,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09996744990348816,
|
|
"step": 53
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1113.0,
|
|
"completions/max_terminated_length": 1113.0,
|
|
"completions/mean_length": 128.3984375,
|
|
"completions/mean_terminated_length": 128.3984375,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"epoch": 0.3456,
|
|
"grad_norm": 0.013177572749555111,
|
|
"learning_rate": 2.0634920634920634e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 44649929.0,
|
|
"reward": 1.010054111480713,
|
|
"reward_std": 0.015776732936501503,
|
|
"rewards/accuracy_reward": 0.1569010466337204,
|
|
"rewards/brier_reward": 0.8632051348686218,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09996744990348816,
|
|
"step": 54
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 494.0,
|
|
"completions/max_terminated_length": 494.0,
|
|
"completions/mean_length": 133.80795288085938,
|
|
"completions/mean_terminated_length": 133.80795288085938,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"epoch": 0.352,
|
|
"grad_norm": 0.014343966729938984,
|
|
"learning_rate": 1.984126984126984e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 45384034.0,
|
|
"reward": 1.0084846019744873,
|
|
"reward_std": 0.01659807190299034,
|
|
"rewards/accuracy_reward": 0.1412760466337204,
|
|
"rewards/brier_reward": 0.8756911158561707,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1129.0,
|
|
"completions/max_terminated_length": 1129.0,
|
|
"completions/mean_length": 136.109375,
|
|
"completions/mean_terminated_length": 136.109375,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"epoch": 0.3584,
|
|
"grad_norm": 0.013693872839212418,
|
|
"learning_rate": 1.904761904761905e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 46122690.0,
|
|
"reward": 1.0078858137130737,
|
|
"reward_std": 0.017217285931110382,
|
|
"rewards/accuracy_reward": 0.1354166716337204,
|
|
"rewards/brier_reward": 0.8803529143333435,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10003254562616348,
|
|
"step": 56
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 419.0,
|
|
"completions/max_terminated_length": 419.0,
|
|
"completions/mean_length": 135.55859375,
|
|
"completions/mean_terminated_length": 135.55859375,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"epoch": 0.3648,
|
|
"grad_norm": 0.014455785974860191,
|
|
"learning_rate": 1.8253968253968254e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 46858812.0,
|
|
"reward": 1.012557864189148,
|
|
"reward_std": 0.019938761368393898,
|
|
"rewards/accuracy_reward": 0.181640625,
|
|
"rewards/brier_reward": 0.843472957611084,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09990235418081284,
|
|
"step": 57
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 809.0,
|
|
"completions/max_terminated_length": 809.0,
|
|
"completions/mean_length": 138.634765625,
|
|
"completions/mean_terminated_length": 138.634765625,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"epoch": 0.3712,
|
|
"grad_norm": 0.012984626926481724,
|
|
"learning_rate": 1.746031746031746e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 47601963.0,
|
|
"reward": 1.0092716217041016,
|
|
"reward_std": 0.016622129827737808,
|
|
"rewards/accuracy_reward": 0.1490885466337204,
|
|
"rewards/brier_reward": 0.8694527745246887,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 58
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 817.0,
|
|
"completions/max_terminated_length": 817.0,
|
|
"completions/mean_length": 137.640625,
|
|
"completions/mean_terminated_length": 137.640625,
|
|
"completions/min_length": 63.0,
|
|
"completions/min_terminated_length": 63.0,
|
|
"epoch": 0.3776,
|
|
"grad_norm": 0.012412709183990955,
|
|
"learning_rate": 1.6666666666666667e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 48338859.0,
|
|
"reward": 1.006111979484558,
|
|
"reward_std": 0.01653250865638256,
|
|
"rewards/accuracy_reward": 0.1178385391831398,
|
|
"rewards/brier_reward": 0.894383430480957,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10006511211395264,
|
|
"step": 59
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 433.0,
|
|
"completions/max_terminated_length": 433.0,
|
|
"completions/mean_length": 137.54232788085938,
|
|
"completions/mean_terminated_length": 137.54232788085938,
|
|
"completions/min_length": 63.0,
|
|
"completions/min_terminated_length": 63.0,
|
|
"epoch": 0.384,
|
|
"grad_norm": 0.013855542987585068,
|
|
"learning_rate": 1.5873015873015873e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 49078244.0,
|
|
"reward": 1.0092642307281494,
|
|
"reward_std": 0.018371157348155975,
|
|
"rewards/accuracy_reward": 0.1490885466337204,
|
|
"rewards/brier_reward": 0.8694377541542053,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.384,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 584.375,
|
|
"eval_completions/max_terminated_length": 584.375,
|
|
"eval_completions/mean_length": 143.89625930786133,
|
|
"eval_completions/mean_terminated_length": 143.89625930786133,
|
|
"eval_completions/min_length": 73.75,
|
|
"eval_completions/min_terminated_length": 73.75,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 49078244.0,
|
|
"eval_reward": 1.0098019689321518,
|
|
"eval_reward_std": 0.030740282498300076,
|
|
"eval_rewards/accuracy_reward": 0.154296875,
|
|
"eval_rewards/brier_reward": 0.8653050512075424,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.09991211164742708,
|
|
"eval_runtime": 51.9353,
|
|
"eval_samples_per_second": 19.255,
|
|
"eval_steps_per_second": 0.154,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 938.0,
|
|
"completions/max_terminated_length": 938.0,
|
|
"completions/mean_length": 143.8619842529297,
|
|
"completions/mean_terminated_length": 143.8619842529297,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"epoch": 0.3904,
|
|
"grad_norm": 0.014213998802006245,
|
|
"learning_rate": 1.507936507936508e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 49836384.0,
|
|
"reward": 1.012863039970398,
|
|
"reward_std": 0.02077551931142807,
|
|
"rewards/accuracy_reward": 0.185546875,
|
|
"rewards/brier_reward": 0.840177059173584,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10003256052732468,
|
|
"step": 61
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 657.0,
|
|
"completions/max_terminated_length": 657.0,
|
|
"completions/mean_length": 144.2779998779297,
|
|
"completions/mean_terminated_length": 144.2779998779297,
|
|
"completions/min_length": 70.0,
|
|
"completions/min_terminated_length": 70.0,
|
|
"epoch": 0.3968,
|
|
"grad_norm": 0.012943128123879433,
|
|
"learning_rate": 1.4285714285714286e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 50587179.0,
|
|
"reward": 1.0079636573791504,
|
|
"reward_std": 0.01760336197912693,
|
|
"rewards/accuracy_reward": 0.13671875,
|
|
"rewards/brier_reward": 0.8792063593864441,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.0999348983168602,
|
|
"step": 62
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1316.0,
|
|
"completions/mean_length": 149.70443725585938,
|
|
"completions/mean_terminated_length": 147.80064392089844,
|
|
"completions/min_length": 64.0,
|
|
"completions/min_terminated_length": 64.0,
|
|
"epoch": 0.4032,
|
|
"grad_norm": 0.015729162842035294,
|
|
"learning_rate": 1.3492063492063493e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 51344197.0,
|
|
"reward": 1.0096068382263184,
|
|
"reward_std": 0.02126619964838028,
|
|
"rewards/accuracy_reward": 0.1588541716337204,
|
|
"rewards/brier_reward": 0.8610084056854248,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.0999348983168602,
|
|
"step": 63
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 568.0,
|
|
"completions/max_terminated_length": 568.0,
|
|
"completions/mean_length": 147.91928100585938,
|
|
"completions/mean_terminated_length": 147.91928100585938,
|
|
"completions/min_length": 63.0,
|
|
"completions/min_terminated_length": 63.0,
|
|
"epoch": 0.4096,
|
|
"grad_norm": 0.012818845920264721,
|
|
"learning_rate": 1.26984126984127e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 52097785.0,
|
|
"reward": 1.009861946105957,
|
|
"reward_std": 0.016676973551511765,
|
|
"rewards/accuracy_reward": 0.1549479216337204,
|
|
"rewards/brier_reward": 0.8647739887237549,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 64
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 526.0,
|
|
"completions/max_terminated_length": 526.0,
|
|
"completions/mean_length": 145.8190155029297,
|
|
"completions/mean_terminated_length": 145.8190155029297,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.416,
|
|
"grad_norm": 0.012777054682374,
|
|
"learning_rate": 1.1904761904761906e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 52853979.0,
|
|
"reward": 1.010127305984497,
|
|
"reward_std": 0.018250633031129837,
|
|
"rewards/accuracy_reward": 0.1575520783662796,
|
|
"rewards/brier_reward": 0.8627002239227295,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.09996744990348816,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 522.0,
|
|
"completions/max_terminated_length": 522.0,
|
|
"completions/mean_length": 148.521484375,
|
|
"completions/mean_terminated_length": 148.521484375,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"epoch": 0.4224,
|
|
"grad_norm": 0.012132398784160614,
|
|
"learning_rate": 1.111111111111111e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 53609156.0,
|
|
"reward": 1.0094685554504395,
|
|
"reward_std": 0.01700403541326523,
|
|
"rewards/accuracy_reward": 0.1510416716337204,
|
|
"rewards/brier_reward": 0.8678932189941406,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 66
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1998.0,
|
|
"completions/mean_length": 154.640625,
|
|
"completions/mean_terminated_length": 150.83702087402344,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"epoch": 0.4288,
|
|
"grad_norm": 0.017242759466171265,
|
|
"learning_rate": 1.0317460317460317e-06,
|
|
"loss": 0.0027,
|
|
"num_tokens": 54371588.0,
|
|
"reward": 1.006808876991272,
|
|
"reward_std": 0.024172242730855942,
|
|
"rewards/accuracy_reward": 0.1438802033662796,
|
|
"rewards/brier_reward": 0.8716884255409241,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 0.998046875,
|
|
"rewards/mean_confidence_reward": 0.09973958879709244,
|
|
"step": 67
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 563.0,
|
|
"completions/mean_length": 155.50326538085938,
|
|
"completions/mean_terminated_length": 153.60325622558594,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"epoch": 0.4352,
|
|
"grad_norm": 0.012770230881869793,
|
|
"learning_rate": 9.523809523809525e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 55139729.0,
|
|
"reward": 1.0081652402877808,
|
|
"reward_std": 0.01910454034805298,
|
|
"rewards/accuracy_reward": 0.14453125,
|
|
"rewards/brier_reward": 0.872448205947876,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.0999348983168602,
|
|
"step": 68
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 427.0,
|
|
"completions/max_terminated_length": 427.0,
|
|
"completions/mean_length": 148.45639038085938,
|
|
"completions/mean_terminated_length": 148.45639038085938,
|
|
"completions/min_length": 64.0,
|
|
"completions/min_terminated_length": 64.0,
|
|
"epoch": 0.4416,
|
|
"grad_norm": 0.012311199679970741,
|
|
"learning_rate": 8.73015873015873e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 55896774.0,
|
|
"reward": 1.0073039531707764,
|
|
"reward_std": 0.017514243721961975,
|
|
"rewards/accuracy_reward": 0.1295572966337204,
|
|
"rewards/brier_reward": 0.8850483894348145,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 69
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2121.0,
|
|
"completions/mean_length": 159.736328125,
|
|
"completions/mean_terminated_length": 157.83908081054688,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"epoch": 0.448,
|
|
"grad_norm": 0.01619199477136135,
|
|
"learning_rate": 7.936507936507937e-07,
|
|
"loss": 0.0017,
|
|
"num_tokens": 56671433.0,
|
|
"reward": 1.0060749053955078,
|
|
"reward_std": 0.02096601389348507,
|
|
"rewards/accuracy_reward": 0.1302083283662796,
|
|
"rewards/brier_reward": 0.8832414746284485,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9986979365348816,
|
|
"rewards/mean_confidence_reward": 0.09986979514360428,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 705.0,
|
|
"completions/max_terminated_length": 705.0,
|
|
"completions/mean_length": 152.62631225585938,
|
|
"completions/mean_terminated_length": 152.62631225585938,
|
|
"completions/min_length": 73.0,
|
|
"completions/min_terminated_length": 73.0,
|
|
"epoch": 0.4544,
|
|
"grad_norm": 0.014190653339028358,
|
|
"learning_rate": 7.142857142857143e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 57429307.0,
|
|
"reward": 1.0079669952392578,
|
|
"reward_std": 0.02005232684314251,
|
|
"rewards/accuracy_reward": 0.142578125,
|
|
"rewards/brier_reward": 0.8740048408508301,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 71
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1038.0,
|
|
"completions/mean_length": 154.34115600585938,
|
|
"completions/mean_terminated_length": 152.4403839111328,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.4608,
|
|
"grad_norm": 0.014969422481954098,
|
|
"learning_rate": 6.34920634920635e-07,
|
|
"loss": 0.0011,
|
|
"num_tokens": 58195767.0,
|
|
"reward": 1.0110514163970947,
|
|
"reward_std": 0.021874962374567986,
|
|
"rewards/accuracy_reward": 0.1731770783662796,
|
|
"rewards/brier_reward": 0.8495745062828064,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.0999348983168602,
|
|
"step": 72
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 602.0,
|
|
"completions/max_terminated_length": 602.0,
|
|
"completions/mean_length": 151.42123413085938,
|
|
"completions/mean_terminated_length": 151.42123413085938,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"epoch": 0.4672,
|
|
"grad_norm": 0.01960376277565956,
|
|
"learning_rate": 5.555555555555555e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 58949094.0,
|
|
"reward": 1.01301908493042,
|
|
"reward_std": 0.022884633392095566,
|
|
"rewards/accuracy_reward": 0.1927083283662796,
|
|
"rewards/brier_reward": 0.8339788317680359,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 73
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 818.0,
|
|
"completions/max_terminated_length": 818.0,
|
|
"completions/mean_length": 150.39779663085938,
|
|
"completions/mean_terminated_length": 150.39779663085938,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"epoch": 0.4736,
|
|
"grad_norm": 0.013314046896994114,
|
|
"learning_rate": 4.7619047619047623e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 59704569.0,
|
|
"reward": 1.01156747341156,
|
|
"reward_std": 0.020016394555568695,
|
|
"rewards/accuracy_reward": 0.171875,
|
|
"rewards/brier_reward": 0.8512578010559082,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 74
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0013020833333333703,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 722.0,
|
|
"completions/mean_length": 150.80990600585938,
|
|
"completions/mean_terminated_length": 147.0012969970703,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.48,
|
|
"grad_norm": 0.014407728798687458,
|
|
"learning_rate": 3.9682539682539683e-07,
|
|
"loss": 0.0023,
|
|
"num_tokens": 60460213.0,
|
|
"reward": 1.00529146194458,
|
|
"reward_std": 0.021866794675588608,
|
|
"rewards/accuracy_reward": 0.1223958358168602,
|
|
"rewards/brier_reward": 0.8894872665405273,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 0.9986979365348816,
|
|
"rewards/mean_confidence_reward": 0.09980469942092896,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.48,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 419.375,
|
|
"eval_completions/max_terminated_length": 419.375,
|
|
"eval_completions/mean_length": 151.05611610412598,
|
|
"eval_completions/mean_terminated_length": 151.05611610412598,
|
|
"eval_completions/min_length": 74.375,
|
|
"eval_completions/min_terminated_length": 74.375,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 60460213.0,
|
|
"eval_reward": 1.0096811801195145,
|
|
"eval_reward_std": 0.03064576326869428,
|
|
"eval_rewards/accuracy_reward": 0.1533203125,
|
|
"eval_rewards/brier_reward": 0.8660400286316872,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.10009765718132257,
|
|
"eval_runtime": 40.5088,
|
|
"eval_samples_per_second": 24.686,
|
|
"eval_steps_per_second": 0.197,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 655.0,
|
|
"completions/max_terminated_length": 655.0,
|
|
"completions/mean_length": 155.2819061279297,
|
|
"completions/mean_terminated_length": 155.2819061279297,
|
|
"completions/min_length": 70.0,
|
|
"completions/min_terminated_length": 70.0,
|
|
"epoch": 0.4864,
|
|
"grad_norm": 0.013746381737291813,
|
|
"learning_rate": 3.174603174603175e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 61222470.0,
|
|
"reward": 1.0095341205596924,
|
|
"reward_std": 0.018527645617723465,
|
|
"rewards/accuracy_reward": 0.1516927033662796,
|
|
"rewards/brier_reward": 0.8673732876777649,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 76
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2094.0,
|
|
"completions/mean_length": 158.095703125,
|
|
"completions/mean_terminated_length": 156.1973876953125,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.4928,
|
|
"grad_norm": 0.013259634375572205,
|
|
"learning_rate": 2.3809523809523811e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 61990177.0,
|
|
"reward": 1.0096008777618408,
|
|
"reward_std": 0.02162647619843483,
|
|
"rewards/accuracy_reward": 0.1588541716337204,
|
|
"rewards/brier_reward": 0.860996425151825,
|
|
"rewards/confidence_one_or_zero": 0.0006510416860692203,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.0999348983168602,
|
|
"step": 77
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 554.0,
|
|
"completions/max_terminated_length": 554.0,
|
|
"completions/mean_length": 147.53451538085938,
|
|
"completions/mean_terminated_length": 147.53451538085938,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"epoch": 0.4992,
|
|
"grad_norm": 0.012484057806432247,
|
|
"learning_rate": 1.5873015873015874e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 62746086.0,
|
|
"reward": 1.0106933116912842,
|
|
"reward_std": 0.018854161724448204,
|
|
"rewards/accuracy_reward": 0.1627604216337204,
|
|
"rewards/brier_reward": 0.8586239814758301,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.10013020783662796,
|
|
"step": 78
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0006510416666666297,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1058.0,
|
|
"completions/mean_length": 158.3697967529297,
|
|
"completions/mean_terminated_length": 156.47166442871094,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"epoch": 0.5056,
|
|
"grad_norm": 0.014450405724346638,
|
|
"learning_rate": 7.936507936507937e-08,
|
|
"loss": 0.0011,
|
|
"num_tokens": 63517686.0,
|
|
"reward": 1.010843276977539,
|
|
"reward_std": 0.022183647379279137,
|
|
"rewards/accuracy_reward": 0.1712239533662796,
|
|
"rewards/brier_reward": 0.8511115908622742,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9993489384651184,
|
|
"rewards/mean_confidence_reward": 0.10000000149011612,
|
|
"step": 79
|
|
},
|
|
{
|
|
"epoch": 0.5056,
|
|
"step": 79,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.001828918740804649,
|
|
"train_runtime": 10390.2846,
|
|
"train_samples_per_second": 1.444,
|
|
"train_steps_per_second": 0.008
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 79,
|
|
"num_input_tokens_seen": 63517686,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 60,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 3,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|