1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"eval_steps": 50,
|
||
|
|
"global_step": 312,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.10869140625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1530.8,
|
||
|
|
"completions/mean_length": 466.23115234375,
|
||
|
|
"completions/mean_terminated_length": 335.8650329589844,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.016,
|
||
|
|
"grad_norm": 0.003617351409047842,
|
||
|
|
"learning_rate": 3.1249999999999997e-07,
|
||
|
|
"loss": 0.0435,
|
||
|
|
"num_tokens": 19618239.0,
|
||
|
|
"reward": 0.19849815964698792,
|
||
|
|
"reward_std": 0.3928296208381653,
|
||
|
|
"rewards/accuracy_reward": 0.0673828125,
|
||
|
|
"rewards/brier_reward": 0.11236810982227326,
|
||
|
|
"rewards/confidence_one_or_zero": 0.4076171875,
|
||
|
|
"rewards/format_reward": 0.26376953125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5322783350944519,
|
||
|
|
"rewards/smce": -0.4652600884437561,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0974609375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1532.8,
|
||
|
|
"completions/mean_length": 438.4470703125,
|
||
|
|
"completions/mean_terminated_length": 320.0117980957031,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.032,
|
||
|
|
"grad_norm": 0.00421218853443861,
|
||
|
|
"learning_rate": 6.249999999999999e-07,
|
||
|
|
"loss": 0.0504,
|
||
|
|
"num_tokens": 39208289.0,
|
||
|
|
"reward": 0.2096415013074875,
|
||
|
|
"reward_std": 0.39408687949180604,
|
||
|
|
"rewards/accuracy_reward": 0.06171875,
|
||
|
|
"rewards/brier_reward": 0.11634623855352402,
|
||
|
|
"rewards/confidence_one_or_zero": 0.4392578125,
|
||
|
|
"rewards/format_reward": 0.2923828125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5730387091636657,
|
||
|
|
"rewards/smce": -0.5116680800914765,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05576171875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1515.8,
|
||
|
|
"completions/mean_length": 318.03837890625,
|
||
|
|
"completions/mean_terminated_length": 246.70384826660157,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.048,
|
||
|
|
"grad_norm": 0.004430481232702732,
|
||
|
|
"learning_rate": 9.374999999999999e-07,
|
||
|
|
"loss": 0.0466,
|
||
|
|
"num_tokens": 57513738.0,
|
||
|
|
"reward": 0.37149760127067566,
|
||
|
|
"reward_std": 0.4421126425266266,
|
||
|
|
"rewards/accuracy_reward": 0.10546875,
|
||
|
|
"rewards/brier_reward": 0.1947878420352936,
|
||
|
|
"rewards/confidence_one_or_zero": 0.528125,
|
||
|
|
"rewards/format_reward": 0.5015625,
|
||
|
|
"rewards/mean_confidence_reward": 0.6924709677696228,
|
||
|
|
"rewards/smce": -0.588262677192688,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.02119140625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1389.4,
|
||
|
|
"completions/mean_length": 209.79072265625,
|
||
|
|
"completions/mean_terminated_length": 181.20720825195312,
|
||
|
|
"completions/min_length": 21.0,
|
||
|
|
"completions/min_terminated_length": 21.0,
|
||
|
|
"epoch": 0.064,
|
||
|
|
"grad_norm": 0.002647513523697853,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0374,
|
||
|
|
"num_tokens": 74580395.0,
|
||
|
|
"reward": 0.6562158942222596,
|
||
|
|
"reward_std": 0.4088839590549469,
|
||
|
|
"rewards/accuracy_reward": 0.1923828125,
|
||
|
|
"rewards/brier_reward": 0.3430897116661072,
|
||
|
|
"rewards/confidence_one_or_zero": 0.55888671875,
|
||
|
|
"rewards/format_reward": 0.83701171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.7904770016670227,
|
||
|
|
"rewards/smce": -0.6005512475967407,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.01181640625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1300.0,
|
||
|
|
"completions/mean_length": 161.79404296875,
|
||
|
|
"completions/mean_terminated_length": 145.35975952148436,
|
||
|
|
"completions/min_length": 22.2,
|
||
|
|
"completions/min_terminated_length": 22.2,
|
||
|
|
"epoch": 0.08,
|
||
|
|
"grad_norm": 0.0026642223820090294,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0175,
|
||
|
|
"num_tokens": 91170318.0,
|
||
|
|
"reward": 0.774070394039154,
|
||
|
|
"reward_std": 0.34370283484458924,
|
||
|
|
"rewards/accuracy_reward": 0.2166015625,
|
||
|
|
"rewards/brier_reward": 0.42684093713760374,
|
||
|
|
"rewards/confidence_one_or_zero": 0.48447265625,
|
||
|
|
"rewards/format_reward": 0.959765625,
|
||
|
|
"rewards/mean_confidence_reward": 0.7623114109039306,
|
||
|
|
"rewards/smce": -0.5506982326507568,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00927734375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1212.6,
|
||
|
|
"completions/mean_length": 149.14765625,
|
||
|
|
"completions/mean_terminated_length": 136.1572692871094,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"epoch": 0.096,
|
||
|
|
"grad_norm": 0.002164568053558469,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0148,
|
||
|
|
"num_tokens": 107742198.0,
|
||
|
|
"reward": 0.8744075894355774,
|
||
|
|
"reward_std": 0.29445708990097047,
|
||
|
|
"rewards/accuracy_reward": 0.23955078125,
|
||
|
|
"rewards/brier_reward": 0.5632546305656433,
|
||
|
|
"rewards/confidence_one_or_zero": 0.3158203125,
|
||
|
|
"rewards/format_reward": 0.98310546875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5917321562767028,
|
||
|
|
"rewards/smce": -0.3709741532802582,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0052734375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 912.6,
|
||
|
|
"completions/mean_length": 133.21044921875,
|
||
|
|
"completions/mean_terminated_length": 125.7759017944336,
|
||
|
|
"completions/min_length": 32.4,
|
||
|
|
"completions/min_terminated_length": 32.4,
|
||
|
|
"epoch": 0.112,
|
||
|
|
"grad_norm": 0.0014111875789240003,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0105,
|
||
|
|
"num_tokens": 124215745.0,
|
||
|
|
"reward": 0.9913592576980591,
|
||
|
|
"reward_std": 0.20873564779758452,
|
||
|
|
"rewards/accuracy_reward": 0.28134765625,
|
||
|
|
"rewards/brier_reward": 0.7238480806350708,
|
||
|
|
"rewards/confidence_one_or_zero": 0.25,
|
||
|
|
"rewards/format_reward": 0.99072265625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3475369572639465,
|
||
|
|
"rewards/smce": -0.13201046139001846,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00126953125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 604.0,
|
||
|
|
"completions/mean_length": 123.39462890625,
|
||
|
|
"completions/mean_terminated_length": 121.59987945556641,
|
||
|
|
"completions/min_length": 37.8,
|
||
|
|
"completions/min_terminated_length": 37.8,
|
||
|
|
"epoch": 0.128,
|
||
|
|
"grad_norm": 0.0009458517306484282,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0029,
|
||
|
|
"num_tokens": 140395978.0,
|
||
|
|
"reward": 1.018831992149353,
|
||
|
|
"reward_std": 0.13349860906600952,
|
||
|
|
"rewards/accuracy_reward": 0.27802734375,
|
||
|
|
"rewards/brier_reward": 0.7702086210250855,
|
||
|
|
"rewards/confidence_one_or_zero": 0.20068359375,
|
||
|
|
"rewards/format_reward": 0.99609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.2456991046667099,
|
||
|
|
"rewards/smce": -0.06666604205965995,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00146484375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 626.6,
|
||
|
|
"completions/mean_length": 121.8669921875,
|
||
|
|
"completions/mean_terminated_length": 119.79229125976562,
|
||
|
|
"completions/min_length": 47.4,
|
||
|
|
"completions/min_terminated_length": 47.4,
|
||
|
|
"epoch": 0.144,
|
||
|
|
"grad_norm": 0.0008370755822397768,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0033,
|
||
|
|
"num_tokens": 156594328.0,
|
||
|
|
"reward": 1.0348623275756836,
|
||
|
|
"reward_std": 0.11074777394533157,
|
||
|
|
"rewards/accuracy_reward": 0.3251953125,
|
||
|
|
"rewards/brier_reward": 0.7585193753242493,
|
||
|
|
"rewards/confidence_one_or_zero": 0.1462890625,
|
||
|
|
"rewards/format_reward": 0.997265625,
|
||
|
|
"rewards/mean_confidence_reward": 0.2217879205942154,
|
||
|
|
"rewards/smce": -0.11256366223096848,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00068359375,
|
||
|
|
"completions/max_length": 1290.4,
|
||
|
|
"completions/max_terminated_length": 458.2,
|
||
|
|
"completions/mean_length": 122.57265625,
|
||
|
|
"completions/mean_terminated_length": 121.60626678466797,
|
||
|
|
"completions/min_length": 51.6,
|
||
|
|
"completions/min_terminated_length": 51.6,
|
||
|
|
"epoch": 0.16,
|
||
|
|
"grad_norm": 0.0007780106971040368,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 172870400.0,
|
||
|
|
"reward": 1.0400798797607422,
|
||
|
|
"reward_std": 0.09919532835483551,
|
||
|
|
"rewards/accuracy_reward": 0.31435546875,
|
||
|
|
"rewards/brier_reward": 0.7744772434234619,
|
||
|
|
"rewards/confidence_one_or_zero": 0.07734375,
|
||
|
|
"rewards/format_reward": 0.9986328125,
|
||
|
|
"rewards/mean_confidence_reward": 0.24838550090789796,
|
||
|
|
"rewards/smce": -0.07306429222226143,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 278.0,
|
||
|
|
"eval_completions/max_terminated_length": 278.0,
|
||
|
|
"eval_completions/mean_length": 121.49138259887695,
|
||
|
|
"eval_completions/mean_terminated_length": 121.49138259887695,
|
||
|
|
"eval_completions/min_length": 62.5,
|
||
|
|
"eval_completions/min_terminated_length": 62.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 172870400.0,
|
||
|
|
"eval_reward": 1.028822124004364,
|
||
|
|
"eval_reward_std": 0.16236025094985962,
|
||
|
|
"eval_rewards/accuracy_reward": 0.26171875,
|
||
|
|
"eval_rewards/brier_reward": 0.7987500131130219,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.033203125,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.2767578214406967,
|
||
|
|
"eval_rewards/smce": -0.028251958079636097,
|
||
|
|
"eval_runtime": 10.2637,
|
||
|
|
"eval_samples_per_second": 48.715,
|
||
|
|
"eval_steps_per_second": 0.195,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1320.6,
|
||
|
|
"completions/max_terminated_length": 557.4,
|
||
|
|
"completions/mean_length": 124.48154296875,
|
||
|
|
"completions/mean_terminated_length": 123.65345611572266,
|
||
|
|
"completions/min_length": 44.4,
|
||
|
|
"completions/min_terminated_length": 44.4,
|
||
|
|
"epoch": 0.176,
|
||
|
|
"grad_norm": 0.0009458002750761807,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 189382211.0,
|
||
|
|
"reward": 1.0453623294830323,
|
||
|
|
"reward_std": 0.10711575001478195,
|
||
|
|
"rewards/accuracy_reward": 0.31328125,
|
||
|
|
"rewards/brier_reward": 0.7821379065513611,
|
||
|
|
"rewards/confidence_one_or_zero": 0.03759765625,
|
||
|
|
"rewards/format_reward": 0.99853515625,
|
||
|
|
"rewards/mean_confidence_reward": 0.29168972969055174,
|
||
|
|
"rewards/smce": -0.03230265900492668,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 885.6,
|
||
|
|
"completions/max_terminated_length": 449.0,
|
||
|
|
"completions/mean_length": 127.11484375,
|
||
|
|
"completions/mean_terminated_length": 126.83966674804688,
|
||
|
|
"completions/min_length": 58.6,
|
||
|
|
"completions/min_terminated_length": 58.6,
|
||
|
|
"epoch": 0.192,
|
||
|
|
"grad_norm": 0.0010142099345102906,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 205498683.0,
|
||
|
|
"reward": 1.0576444864273071,
|
||
|
|
"reward_std": 0.12386192083358764,
|
||
|
|
"rewards/accuracy_reward": 0.34736328125,
|
||
|
|
"rewards/brier_reward": 0.7734477996826172,
|
||
|
|
"rewards/confidence_one_or_zero": 0.01337890625,
|
||
|
|
"rewards/format_reward": 0.9990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.34815791845321653,
|
||
|
|
"rewards/smce": -0.04546214714646339,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1063.4,
|
||
|
|
"completions/max_terminated_length": 436.0,
|
||
|
|
"completions/mean_length": 134.46962890625,
|
||
|
|
"completions/mean_terminated_length": 133.785693359375,
|
||
|
|
"completions/min_length": 55.0,
|
||
|
|
"completions/min_terminated_length": 55.0,
|
||
|
|
"epoch": 0.208,
|
||
|
|
"grad_norm": 0.0010471841087564826,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 221907876.0,
|
||
|
|
"reward": 1.0676575660705567,
|
||
|
|
"reward_std": 0.13774634152650833,
|
||
|
|
"rewards/accuracy_reward": 0.36845703125,
|
||
|
|
"rewards/brier_reward": 0.7705541491508484,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0060546875,
|
||
|
|
"rewards/format_reward": 0.9990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3837132155895233,
|
||
|
|
"rewards/smce": -0.027201873436570167,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 835.4,
|
||
|
|
"completions/max_terminated_length": 387.2,
|
||
|
|
"completions/mean_length": 131.84931640625,
|
||
|
|
"completions/mean_terminated_length": 131.43808288574218,
|
||
|
|
"completions/min_length": 58.4,
|
||
|
|
"completions/min_terminated_length": 58.4,
|
||
|
|
"epoch": 0.224,
|
||
|
|
"grad_norm": 0.0010664016008377075,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"num_tokens": 238411197.0,
|
||
|
|
"reward": 1.0667345762252807,
|
||
|
|
"reward_std": 0.12648420929908752,
|
||
|
|
"rewards/accuracy_reward": 0.3685546875,
|
||
|
|
"rewards/brier_reward": 0.7694954991340637,
|
||
|
|
"rewards/confidence_one_or_zero": 0.004296875,
|
||
|
|
"rewards/format_reward": 0.99892578125,
|
||
|
|
"rewards/mean_confidence_reward": 0.38269283771514895,
|
||
|
|
"rewards/smce": -0.035074672847986224,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 882.8,
|
||
|
|
"completions/max_terminated_length": 421.2,
|
||
|
|
"completions/mean_length": 131.6205078125,
|
||
|
|
"completions/mean_terminated_length": 131.34631958007813,
|
||
|
|
"completions/min_length": 58.4,
|
||
|
|
"completions/min_terminated_length": 58.4,
|
||
|
|
"epoch": 0.24,
|
||
|
|
"grad_norm": 0.001074860105291009,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 255010671.0,
|
||
|
|
"reward": 1.0831371784210204,
|
||
|
|
"reward_std": 0.12712678611278533,
|
||
|
|
"rewards/accuracy_reward": 0.4126953125,
|
||
|
|
"rewards/brier_reward": 0.7609473943710328,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0015625,
|
||
|
|
"rewards/format_reward": 0.99921875,
|
||
|
|
"rewards/mean_confidence_reward": 0.39454362392425535,
|
||
|
|
"rewards/smce": -0.06587964631617069,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 596.8,
|
||
|
|
"completions/max_terminated_length": 363.6,
|
||
|
|
"completions/mean_length": 130.2615234375,
|
||
|
|
"completions/mean_terminated_length": 130.1241485595703,
|
||
|
|
"completions/min_length": 57.2,
|
||
|
|
"completions/min_terminated_length": 57.2,
|
||
|
|
"epoch": 0.256,
|
||
|
|
"grad_norm": 0.0011320032645016909,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 271399365.0,
|
||
|
|
"reward": 1.0704259634017945,
|
||
|
|
"reward_std": 0.13221554309129716,
|
||
|
|
"rewards/accuracy_reward": 0.3767578125,
|
||
|
|
"rewards/brier_reward": 0.7702070951461792,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.40313740968704226,
|
||
|
|
"rewards/smce": -0.05625563189387321,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 683.0,
|
||
|
|
"completions/max_terminated_length": 446.2,
|
||
|
|
"completions/mean_length": 133.1107421875,
|
||
|
|
"completions/mean_terminated_length": 132.97392883300782,
|
||
|
|
"completions/min_length": 58.4,
|
||
|
|
"completions/min_terminated_length": 58.4,
|
||
|
|
"epoch": 0.272,
|
||
|
|
"grad_norm": 0.0009479325381107628,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0,
|
||
|
|
"num_tokens": 287728115.0,
|
||
|
|
"reward": 1.0818254709243775,
|
||
|
|
"reward_std": 0.1228899359703064,
|
||
|
|
"rewards/accuracy_reward": 0.40283203125,
|
||
|
|
"rewards/brier_reward": 0.7644219875335694,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.39888736605644226,
|
||
|
|
"rewards/smce": -0.03408620785921812,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1113.6,
|
||
|
|
"completions/max_terminated_length": 411.6,
|
||
|
|
"completions/mean_length": 134.308984375,
|
||
|
|
"completions/mean_terminated_length": 133.48701171875,
|
||
|
|
"completions/min_length": 59.4,
|
||
|
|
"completions/min_terminated_length": 59.4,
|
||
|
|
"epoch": 0.288,
|
||
|
|
"grad_norm": 0.0009760709363035858,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"num_tokens": 304061615.0,
|
||
|
|
"reward": 1.0798555850982665,
|
||
|
|
"reward_std": 0.1159628912806511,
|
||
|
|
"rewards/accuracy_reward": 0.39912109375,
|
||
|
|
"rewards/brier_reward": 0.766473650932312,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99912109375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3930133879184723,
|
||
|
|
"rewards/smce": -0.050054707750678065,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 866.4,
|
||
|
|
"completions/max_terminated_length": 438.0,
|
||
|
|
"completions/mean_length": 136.6109375,
|
||
|
|
"completions/mean_terminated_length": 136.33718872070312,
|
||
|
|
"completions/min_length": 64.4,
|
||
|
|
"completions/min_terminated_length": 64.4,
|
||
|
|
"epoch": 0.304,
|
||
|
|
"grad_norm": 0.0010910023702308536,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 320390463.0,
|
||
|
|
"reward": 1.0854009866714478,
|
||
|
|
"reward_std": 0.11015888154506684,
|
||
|
|
"rewards/accuracy_reward": 0.41083984375,
|
||
|
|
"rewards/brier_reward": 0.7642677545547485,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.39040674567222594,
|
||
|
|
"rewards/smce": -0.0401339016854763,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 650.6,
|
||
|
|
"completions/max_terminated_length": 416.2,
|
||
|
|
"completions/mean_length": 141.7556640625,
|
||
|
|
"completions/mean_terminated_length": 141.61908874511718,
|
||
|
|
"completions/min_length": 61.0,
|
||
|
|
"completions/min_terminated_length": 61.0,
|
||
|
|
"epoch": 0.32,
|
||
|
|
"grad_norm": 0.0009000017307698727,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 336930745.0,
|
||
|
|
"reward": 1.0933893203735352,
|
||
|
|
"reward_std": 0.10807573348283768,
|
||
|
|
"rewards/accuracy_reward": 0.42783203125,
|
||
|
|
"rewards/brier_reward": 0.7655725359916687,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4067640006542206,
|
||
|
|
"rewards/smce": -0.06333717741072178,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 281.5,
|
||
|
|
"eval_completions/max_terminated_length": 281.5,
|
||
|
|
"eval_completions/mean_length": 144.8693618774414,
|
||
|
|
"eval_completions/mean_terminated_length": 144.8693618774414,
|
||
|
|
"eval_completions/min_length": 73.0,
|
||
|
|
"eval_completions/min_terminated_length": 73.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 336930745.0,
|
||
|
|
"eval_reward": 1.0506283044815063,
|
||
|
|
"eval_reward_std": 0.2293572798371315,
|
||
|
|
"eval_rewards/accuracy_reward": 0.32421875,
|
||
|
|
"eval_rewards/brier_reward": 0.7854808866977692,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4080800861120224,
|
||
|
|
"eval_rewards/smce": -0.08443867601454258,
|
||
|
|
"eval_runtime": 10.6049,
|
||
|
|
"eval_samples_per_second": 47.148,
|
||
|
|
"eval_steps_per_second": 0.189,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 881.4,
|
||
|
|
"completions/max_terminated_length": 410.6,
|
||
|
|
"completions/mean_length": 146.67197265625,
|
||
|
|
"completions/mean_terminated_length": 146.263525390625,
|
||
|
|
"completions/min_length": 63.2,
|
||
|
|
"completions/min_terminated_length": 63.2,
|
||
|
|
"epoch": 0.336,
|
||
|
|
"grad_norm": 0.000741802214179188,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 353155098.0,
|
||
|
|
"reward": 1.0966195583343505,
|
||
|
|
"reward_std": 0.10717584788799286,
|
||
|
|
"rewards/accuracy_reward": 0.43037109375,
|
||
|
|
"rewards/brier_reward": 0.7681578040122986,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.41693392395973206,
|
||
|
|
"rewards/smce": -0.04899918362498283,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 412.8,
|
||
|
|
"completions/max_terminated_length": 412.8,
|
||
|
|
"completions/mean_length": 152.62900390625,
|
||
|
|
"completions/mean_terminated_length": 152.62900390625,
|
||
|
|
"completions/min_length": 61.6,
|
||
|
|
"completions/min_terminated_length": 61.6,
|
||
|
|
"epoch": 0.352,
|
||
|
|
"grad_norm": 0.0008869222365319729,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 369978435.0,
|
||
|
|
"reward": 1.0743722677230836,
|
||
|
|
"reward_std": 0.12426654249429703,
|
||
|
|
"rewards/accuracy_reward": 0.380078125,
|
||
|
|
"rewards/brier_reward": 0.7729269146919251,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4185358822345734,
|
||
|
|
"rewards/smce": -0.03968314975500107,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 874.8,
|
||
|
|
"completions/max_terminated_length": 451.2,
|
||
|
|
"completions/mean_length": 153.2439453125,
|
||
|
|
"completions/mean_terminated_length": 152.97373657226564,
|
||
|
|
"completions/min_length": 59.6,
|
||
|
|
"completions/min_terminated_length": 59.6,
|
||
|
|
"epoch": 0.368,
|
||
|
|
"grad_norm": 0.000924785912502557,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 386613125.0,
|
||
|
|
"reward": 1.0886813402175903,
|
||
|
|
"reward_std": 0.11303802132606507,
|
||
|
|
"rewards/accuracy_reward": 0.40703125,
|
||
|
|
"rewards/brier_reward": 0.7744149088859558,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.42386980056762696,
|
||
|
|
"rewards/smce": -0.03596003744751215,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 361.6,
|
||
|
|
"completions/max_terminated_length": 361.6,
|
||
|
|
"completions/mean_length": 149.928125,
|
||
|
|
"completions/mean_terminated_length": 149.928125,
|
||
|
|
"completions/min_length": 68.0,
|
||
|
|
"completions/min_terminated_length": 68.0,
|
||
|
|
"epoch": 0.384,
|
||
|
|
"grad_norm": 0.0007946203113533556,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 403004901.0,
|
||
|
|
"reward": 1.0991857767105102,
|
||
|
|
"reward_std": 0.10030071586370468,
|
||
|
|
"rewards/accuracy_reward": 0.42626953125,
|
||
|
|
"rewards/brier_reward": 0.7759941816329956,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.41403782963752744,
|
||
|
|
"rewards/smce": -0.037953479960560796,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 652.4,
|
||
|
|
"completions/max_terminated_length": 504.6,
|
||
|
|
"completions/mean_length": 149.6595703125,
|
||
|
|
"completions/mean_terminated_length": 149.52425842285157,
|
||
|
|
"completions/min_length": 64.2,
|
||
|
|
"completions/min_terminated_length": 64.2,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"grad_norm": 0.0008367216214537621,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 419573863.0,
|
||
|
|
"reward": 1.0881041288375854,
|
||
|
|
"reward_std": 0.10986830145120621,
|
||
|
|
"rewards/accuracy_reward": 0.40126953125,
|
||
|
|
"rewards/brier_reward": 0.7783864617347718,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.41706185340881347,
|
||
|
|
"rewards/smce": -0.03350854460150003,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 441.4,
|
||
|
|
"completions/max_terminated_length": 441.4,
|
||
|
|
"completions/mean_length": 151.35322265625,
|
||
|
|
"completions/mean_terminated_length": 151.35322265625,
|
||
|
|
"completions/min_length": 70.0,
|
||
|
|
"completions/min_terminated_length": 70.0,
|
||
|
|
"epoch": 0.416,
|
||
|
|
"grad_norm": 0.0007317252457141876,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 436004904.0,
|
||
|
|
"reward": 1.0960778713226318,
|
||
|
|
"reward_std": 0.106614151597023,
|
||
|
|
"rewards/accuracy_reward": 0.41640625,
|
||
|
|
"rewards/brier_reward": 0.7772732257843018,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.41116663813591003,
|
||
|
|
"rewards/smce": -0.015245692990720271,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 654.4,
|
||
|
|
"completions/max_terminated_length": 417.6,
|
||
|
|
"completions/mean_length": 154.1056640625,
|
||
|
|
"completions/mean_terminated_length": 153.97059631347656,
|
||
|
|
"completions/min_length": 72.0,
|
||
|
|
"completions/min_terminated_length": 72.0,
|
||
|
|
"epoch": 0.432,
|
||
|
|
"grad_norm": 0.0007601842517033219,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 452597282.0,
|
||
|
|
"reward": 1.1045128583908081,
|
||
|
|
"reward_std": 0.10084618777036666,
|
||
|
|
"rewards/accuracy_reward": 0.43818359375,
|
||
|
|
"rewards/brier_reward": 0.7752175211906434,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4205751955509186,
|
||
|
|
"rewards/smce": -0.04278548639267683,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 435.2,
|
||
|
|
"completions/max_terminated_length": 435.2,
|
||
|
|
"completions/mean_length": 157.68974609375,
|
||
|
|
"completions/mean_terminated_length": 157.68974609375,
|
||
|
|
"completions/min_length": 72.8,
|
||
|
|
"completions/min_terminated_length": 72.8,
|
||
|
|
"epoch": 0.448,
|
||
|
|
"grad_norm": 0.0006946580251678824,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 469164825.0,
|
||
|
|
"reward": 1.1021424531936646,
|
||
|
|
"reward_std": 0.10896257162094117,
|
||
|
|
"rewards/accuracy_reward": 0.4326171875,
|
||
|
|
"rewards/brier_reward": 0.7751887559890747,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4384663462638855,
|
||
|
|
"rewards/smce": -0.03424236755818129,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 879.2,
|
||
|
|
"completions/max_terminated_length": 420.6,
|
||
|
|
"completions/mean_length": 159.316796875,
|
||
|
|
"completions/mean_terminated_length": 159.04764404296876,
|
||
|
|
"completions/min_length": 72.6,
|
||
|
|
"completions/min_terminated_length": 72.6,
|
||
|
|
"epoch": 0.464,
|
||
|
|
"grad_norm": 0.0008301216876134276,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 485967045.0,
|
||
|
|
"reward": 1.076231551170349,
|
||
|
|
"reward_std": 0.11073880642652512,
|
||
|
|
"rewards/accuracy_reward": 0.3748046875,
|
||
|
|
"rewards/brier_reward": 0.7852112293243408,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4326315402984619,
|
||
|
|
"rewards/smce": -0.07358203530311584,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 629.8,
|
||
|
|
"completions/max_terminated_length": 417.4,
|
||
|
|
"completions/mean_length": 160.202734375,
|
||
|
|
"completions/mean_terminated_length": 160.06853942871095,
|
||
|
|
"completions/min_length": 74.2,
|
||
|
|
"completions/min_terminated_length": 74.2,
|
||
|
|
"epoch": 0.48,
|
||
|
|
"grad_norm": 0.0008279711473733187,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 502655553.0,
|
||
|
|
"reward": 1.106163740158081,
|
||
|
|
"reward_std": 0.10428445488214493,
|
||
|
|
"rewards/accuracy_reward": 0.4455078125,
|
||
|
|
"rewards/brier_reward": 0.7723424196243286,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.41151692271232604,
|
||
|
|
"rewards/smce": -0.053282588720321655,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 348.5,
|
||
|
|
"eval_completions/max_terminated_length": 348.5,
|
||
|
|
"eval_completions/mean_length": 159.40221405029297,
|
||
|
|
"eval_completions/mean_terminated_length": 159.40221405029297,
|
||
|
|
"eval_completions/min_length": 76.5,
|
||
|
|
"eval_completions/min_terminated_length": 76.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 502655553.0,
|
||
|
|
"eval_reward": 1.0689831376075745,
|
||
|
|
"eval_reward_std": 0.23300867527723312,
|
||
|
|
"eval_rewards/accuracy_reward": 0.345703125,
|
||
|
|
"eval_rewards/brier_reward": 0.7980294525623322,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4022851586341858,
|
||
|
|
"eval_rewards/smce": -0.05767090246081352,
|
||
|
|
"eval_runtime": 11.9118,
|
||
|
|
"eval_samples_per_second": 41.975,
|
||
|
|
"eval_steps_per_second": 0.168,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 404.0,
|
||
|
|
"completions/max_terminated_length": 404.0,
|
||
|
|
"completions/mean_length": 157.42607421875,
|
||
|
|
"completions/mean_terminated_length": 157.42607421875,
|
||
|
|
"completions/min_length": 72.0,
|
||
|
|
"completions/min_terminated_length": 72.0,
|
||
|
|
"epoch": 0.496,
|
||
|
|
"grad_norm": 0.0008031203760765493,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 519575436.0,
|
||
|
|
"reward": 1.1064825534820557,
|
||
|
|
"reward_std": 0.10670529305934906,
|
||
|
|
"rewards/accuracy_reward": 0.44521484375,
|
||
|
|
"rewards/brier_reward": 0.7713125824928284,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.42044044137001035,
|
||
|
|
"rewards/smce": -0.03563190307468176,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 376.8,
|
||
|
|
"completions/max_terminated_length": 376.8,
|
||
|
|
"completions/mean_length": 156.19970703125,
|
||
|
|
"completions/mean_terminated_length": 156.19970703125,
|
||
|
|
"completions/min_length": 78.0,
|
||
|
|
"completions/min_terminated_length": 78.0,
|
||
|
|
"epoch": 0.512,
|
||
|
|
"grad_norm": 0.0009704737458378077,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0004,
|
||
|
|
"num_tokens": 536320585.0,
|
||
|
|
"reward": 1.1233646392822265,
|
||
|
|
"reward_std": 0.10814391821622849,
|
||
|
|
"rewards/accuracy_reward": 0.47216796875,
|
||
|
|
"rewards/brier_reward": 0.7789788365364074,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4404987394809723,
|
||
|
|
"rewards/smce": -0.04320650082081556,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 869.6,
|
||
|
|
"completions/max_terminated_length": 417.2,
|
||
|
|
"completions/mean_length": 160.86787109375,
|
||
|
|
"completions/mean_terminated_length": 160.59920349121094,
|
||
|
|
"completions/min_length": 84.0,
|
||
|
|
"completions/min_terminated_length": 84.0,
|
||
|
|
"epoch": 0.528,
|
||
|
|
"grad_norm": 0.0009864222956821322,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 552997408.0,
|
||
|
|
"reward": 1.1117675065994264,
|
||
|
|
"reward_std": 0.12492350935935974,
|
||
|
|
"rewards/accuracy_reward": 0.4421875,
|
||
|
|
"rewards/brier_reward": 0.7852141976356506,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.47042285203933715,
|
||
|
|
"rewards/smce": -0.035746574960649015,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 883.6,
|
||
|
|
"completions/max_terminated_length": 435.2,
|
||
|
|
"completions/mean_length": 159.598828125,
|
||
|
|
"completions/mean_terminated_length": 159.33026428222655,
|
||
|
|
"completions/min_length": 80.4,
|
||
|
|
"completions/min_terminated_length": 80.4,
|
||
|
|
"epoch": 0.544,
|
||
|
|
"grad_norm": 0.0008369262213818729,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 569795284.0,
|
||
|
|
"reward": 1.1314317226409911,
|
||
|
|
"reward_std": 0.11883504688739777,
|
||
|
|
"rewards/accuracy_reward": 0.49619140625,
|
||
|
|
"rewards/brier_reward": 0.7696640849113464,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.48162569403648375,
|
||
|
|
"rewards/smce": -0.027976423129439355,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 354.2,
|
||
|
|
"completions/max_terminated_length": 354.2,
|
||
|
|
"completions/mean_length": 153.4052734375,
|
||
|
|
"completions/mean_terminated_length": 153.4052734375,
|
||
|
|
"completions/min_length": 73.0,
|
||
|
|
"completions/min_terminated_length": 73.0,
|
||
|
|
"epoch": 0.56,
|
||
|
|
"grad_norm": 0.0009207469411194324,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 586187562.0,
|
||
|
|
"reward": 1.099264669418335,
|
||
|
|
"reward_std": 0.12063608765602112,
|
||
|
|
"rewards/accuracy_reward": 0.41640625,
|
||
|
|
"rewards/brier_reward": 0.7878692507743835,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4717119216918945,
|
||
|
|
"rewards/smce": -0.056493572890758514,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 830.0,
|
||
|
|
"completions/max_terminated_length": 334.2,
|
||
|
|
"completions/mean_length": 150.38310546875,
|
||
|
|
"completions/mean_terminated_length": 150.1125,
|
||
|
|
"completions/min_length": 72.2,
|
||
|
|
"completions/min_terminated_length": 72.2,
|
||
|
|
"epoch": 0.576,
|
||
|
|
"grad_norm": 0.000673173344694078,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 602914109.0,
|
||
|
|
"reward": 1.10039484500885,
|
||
|
|
"reward_std": 0.09124593585729598,
|
||
|
|
"rewards/accuracy_reward": 0.42099609375,
|
||
|
|
"rewards/brier_reward": 0.7848548769950867,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4232636749744415,
|
||
|
|
"rewards/smce": -0.04866776242852211,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 390.8,
|
||
|
|
"completions/max_terminated_length": 390.8,
|
||
|
|
"completions/mean_length": 148.59873046875,
|
||
|
|
"completions/mean_terminated_length": 148.59873046875,
|
||
|
|
"completions/min_length": 74.2,
|
||
|
|
"completions/min_terminated_length": 74.2,
|
||
|
|
"epoch": 0.592,
|
||
|
|
"grad_norm": 0.0007362031610682607,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 619603472.0,
|
||
|
|
"reward": 1.109256672859192,
|
||
|
|
"reward_std": 0.08407505601644516,
|
||
|
|
"rewards/accuracy_reward": 0.4404296875,
|
||
|
|
"rewards/brier_reward": 0.7812385320663452,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.41702051758766173,
|
||
|
|
"rewards/smce": -0.03155655637383461,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 427.6,
|
||
|
|
"completions/max_terminated_length": 427.6,
|
||
|
|
"completions/mean_length": 152.87138671875,
|
||
|
|
"completions/mean_terminated_length": 152.87138671875,
|
||
|
|
"completions/min_length": 78.6,
|
||
|
|
"completions/min_terminated_length": 78.6,
|
||
|
|
"epoch": 0.608,
|
||
|
|
"grad_norm": 0.0009035294642671943,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 636168363.0,
|
||
|
|
"reward": 1.1182846546173095,
|
||
|
|
"reward_std": 0.09603894054889679,
|
||
|
|
"rewards/accuracy_reward": 0.44208984375,
|
||
|
|
"rewards/brier_reward": 0.7972625851631164,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.43902148604393004,
|
||
|
|
"rewards/smce": -0.02784018460661173,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 896.8,
|
||
|
|
"completions/max_terminated_length": 418.2,
|
||
|
|
"completions/mean_length": 159.56494140625,
|
||
|
|
"completions/mean_terminated_length": 159.29615173339843,
|
||
|
|
"completions/min_length": 79.8,
|
||
|
|
"completions/min_terminated_length": 79.8,
|
||
|
|
"epoch": 0.624,
|
||
|
|
"grad_norm": 0.0011967478785663843,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 653146212.0,
|
||
|
|
"reward": 1.1095570087432862,
|
||
|
|
"reward_std": 0.11243704110383987,
|
||
|
|
"rewards/accuracy_reward": 0.43642578125,
|
||
|
|
"rewards/brier_reward": 0.7865937709808349,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.46078904867172243,
|
||
|
|
"rewards/smce": -0.037111565470695496,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 474.2,
|
||
|
|
"completions/max_terminated_length": 474.2,
|
||
|
|
"completions/mean_length": 163.27529296875,
|
||
|
|
"completions/mean_terminated_length": 163.27529296875,
|
||
|
|
"completions/min_length": 80.4,
|
||
|
|
"completions/min_terminated_length": 80.4,
|
||
|
|
"epoch": 0.64,
|
||
|
|
"grad_norm": 0.0007251665811054409,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 670160839.0,
|
||
|
|
"reward": 1.1448447942733764,
|
||
|
|
"reward_std": 0.09686506688594818,
|
||
|
|
"rewards/accuracy_reward": 0.5076171875,
|
||
|
|
"rewards/brier_reward": 0.7882843375205993,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.466916024684906,
|
||
|
|
"rewards/smce": -0.06212771981954575,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 316.5,
|
||
|
|
"eval_completions/max_terminated_length": 316.5,
|
||
|
|
"eval_completions/mean_length": 167.25476837158203,
|
||
|
|
"eval_completions/mean_terminated_length": 167.25476837158203,
|
||
|
|
"eval_completions/min_length": 88.0,
|
||
|
|
"eval_completions/min_terminated_length": 88.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 670160839.0,
|
||
|
|
"eval_reward": 1.0876256227493286,
|
||
|
|
"eval_reward_std": 0.26613427698612213,
|
||
|
|
"eval_rewards/accuracy_reward": 0.384765625,
|
||
|
|
"eval_rewards/brier_reward": 0.7975634634494781,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.45537109673023224,
|
||
|
|
"eval_rewards/smce": -0.0707861389964819,
|
||
|
|
"eval_runtime": 10.9563,
|
||
|
|
"eval_samples_per_second": 45.636,
|
||
|
|
"eval_steps_per_second": 0.183,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 671.4,
|
||
|
|
"completions/max_terminated_length": 449.6,
|
||
|
|
"completions/mean_length": 164.71435546875,
|
||
|
|
"completions/mean_terminated_length": 164.58023071289062,
|
||
|
|
"completions/min_length": 81.2,
|
||
|
|
"completions/min_terminated_length": 81.2,
|
||
|
|
"epoch": 0.656,
|
||
|
|
"grad_norm": 0.0010223239660263062,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 686704058.0,
|
||
|
|
"reward": 1.0958014249801635,
|
||
|
|
"reward_std": 0.11311615407466888,
|
||
|
|
"rewards/accuracy_reward": 0.41611328125,
|
||
|
|
"rewards/brier_reward": 0.7816610336303711,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.47294239401817323,
|
||
|
|
"rewards/smce": -0.060746995359659196,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 437.4,
|
||
|
|
"completions/max_terminated_length": 437.4,
|
||
|
|
"completions/mean_length": 169.5923828125,
|
||
|
|
"completions/mean_terminated_length": 169.5923828125,
|
||
|
|
"completions/min_length": 86.2,
|
||
|
|
"completions/min_terminated_length": 86.2,
|
||
|
|
"epoch": 0.672,
|
||
|
|
"grad_norm": 0.0006438858108595014,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 703354124.0,
|
||
|
|
"reward": 1.1204289674758912,
|
||
|
|
"reward_std": 0.08563904613256454,
|
||
|
|
"rewards/accuracy_reward": 0.45576171875,
|
||
|
|
"rewards/brier_reward": 0.7892528295516967,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.42501270174980166,
|
||
|
|
"rewards/smce": -0.04157466329634189,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 689.0,
|
||
|
|
"completions/max_terminated_length": 480.4,
|
||
|
|
"completions/mean_length": 176.6537109375,
|
||
|
|
"completions/mean_terminated_length": 176.5207946777344,
|
||
|
|
"completions/min_length": 87.2,
|
||
|
|
"completions/min_terminated_length": 87.2,
|
||
|
|
"epoch": 0.688,
|
||
|
|
"grad_norm": 0.0008102591382339597,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 720116978.0,
|
||
|
|
"reward": 1.1223333358764649,
|
||
|
|
"reward_std": 0.09338834285736083,
|
||
|
|
"rewards/accuracy_reward": 0.4533203125,
|
||
|
|
"rewards/brier_reward": 0.7958316683769227,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4228310644626617,
|
||
|
|
"rewards/smce": -0.04388436172157526,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 685.0,
|
||
|
|
"completions/max_terminated_length": 484.0,
|
||
|
|
"completions/mean_length": 179.18369140625,
|
||
|
|
"completions/mean_terminated_length": 179.05135498046874,
|
||
|
|
"completions/min_length": 93.6,
|
||
|
|
"completions/min_terminated_length": 93.6,
|
||
|
|
"epoch": 0.704,
|
||
|
|
"grad_norm": 0.0007676195236854255,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 736817963.0,
|
||
|
|
"reward": 1.131119704246521,
|
||
|
|
"reward_std": 0.0943239152431488,
|
||
|
|
"rewards/accuracy_reward": 0.47373046875,
|
||
|
|
"rewards/brier_reward": 0.7917332649230957,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.44699220061302186,
|
||
|
|
"rewards/smce": -0.03127504363656044,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 741.0,
|
||
|
|
"completions/max_terminated_length": 503.2,
|
||
|
|
"completions/mean_length": 180.8234375,
|
||
|
|
"completions/mean_terminated_length": 180.69107055664062,
|
||
|
|
"completions/min_length": 90.0,
|
||
|
|
"completions/min_terminated_length": 90.0,
|
||
|
|
"epoch": 0.72,
|
||
|
|
"grad_norm": 0.0009789899922907352,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 753679451.0,
|
||
|
|
"reward": 1.1428563594818115,
|
||
|
|
"reward_std": 0.10433020591735839,
|
||
|
|
"rewards/accuracy_reward": 0.4857421875,
|
||
|
|
"rewards/brier_reward": 0.8039116024971008,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.504841786623001,
|
||
|
|
"rewards/smce": -0.0384433601051569,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 483.6,
|
||
|
|
"completions/max_terminated_length": 483.6,
|
||
|
|
"completions/mean_length": 183.2181640625,
|
||
|
|
"completions/mean_terminated_length": 183.2181640625,
|
||
|
|
"completions/min_length": 85.2,
|
||
|
|
"completions/min_terminated_length": 85.2,
|
||
|
|
"epoch": 0.736,
|
||
|
|
"grad_norm": 0.0009930457454174757,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 770495189.0,
|
||
|
|
"reward": 1.1361095428466796,
|
||
|
|
"reward_std": 0.123087178170681,
|
||
|
|
"rewards/accuracy_reward": 0.49052734375,
|
||
|
|
"rewards/brier_reward": 0.7868128776550293,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5383701086044311,
|
||
|
|
"rewards/smce": -0.05024341568350792,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 627.2,
|
||
|
|
"completions/max_terminated_length": 419.0,
|
||
|
|
"completions/mean_length": 185.18837890625,
|
||
|
|
"completions/mean_terminated_length": 185.05672302246094,
|
||
|
|
"completions/min_length": 87.8,
|
||
|
|
"completions/min_terminated_length": 87.8,
|
||
|
|
"epoch": 0.752,
|
||
|
|
"grad_norm": 0.0009987555677071214,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 787618718.0,
|
||
|
|
"reward": 1.1308838367462157,
|
||
|
|
"reward_std": 0.11320770680904388,
|
||
|
|
"rewards/accuracy_reward": 0.47373046875,
|
||
|
|
"rewards/brier_reward": 0.792686128616333,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4944267630577087,
|
||
|
|
"rewards/smce": -0.04552147686481476,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 481.4,
|
||
|
|
"completions/max_terminated_length": 481.4,
|
||
|
|
"completions/mean_length": 191.2140625,
|
||
|
|
"completions/mean_terminated_length": 191.2140625,
|
||
|
|
"completions/min_length": 95.6,
|
||
|
|
"completions/min_terminated_length": 95.6,
|
||
|
|
"epoch": 0.768,
|
||
|
|
"grad_norm": 0.0008614786202087998,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 804509454.0,
|
||
|
|
"reward": 1.136540699005127,
|
||
|
|
"reward_std": 0.094244784116745,
|
||
|
|
"rewards/accuracy_reward": 0.473828125,
|
||
|
|
"rewards/brier_reward": 0.8010059595108032,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.47049121260643006,
|
||
|
|
"rewards/smce": -0.017536511179059744,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 642.6,
|
||
|
|
"completions/max_terminated_length": 434.0,
|
||
|
|
"completions/mean_length": 192.61494140625,
|
||
|
|
"completions/mean_terminated_length": 192.4836853027344,
|
||
|
|
"completions/min_length": 101.0,
|
||
|
|
"completions/min_terminated_length": 101.0,
|
||
|
|
"epoch": 0.784,
|
||
|
|
"grad_norm": 0.000979123986326158,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 821656199.0,
|
||
|
|
"reward": 1.1346184253692626,
|
||
|
|
"reward_std": 0.09833287298679352,
|
||
|
|
"rewards/accuracy_reward": 0.4875,
|
||
|
|
"rewards/brier_reward": 0.7900451183319092,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4683769643306732,
|
||
|
|
"rewards/smce": -0.08211531937122345,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 897.8,
|
||
|
|
"completions/max_terminated_length": 464.0,
|
||
|
|
"completions/mean_length": 194.98642578125,
|
||
|
|
"completions/mean_terminated_length": 194.72418823242188,
|
||
|
|
"completions/min_length": 98.2,
|
||
|
|
"completions/min_terminated_length": 98.2,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 0.0008208116050809622,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 838663420.0,
|
||
|
|
"reward": 1.149678373336792,
|
||
|
|
"reward_std": 0.10992196351289749,
|
||
|
|
"rewards/accuracy_reward": 0.5005859375,
|
||
|
|
"rewards/brier_reward": 0.8018845319747925,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4957675993442535,
|
||
|
|
"rewards/smce": -0.029193690791726113,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 396.5,
|
||
|
|
"eval_completions/max_terminated_length": 396.5,
|
||
|
|
"eval_completions/mean_length": 201.88136291503906,
|
||
|
|
"eval_completions/mean_terminated_length": 201.88136291503906,
|
||
|
|
"eval_completions/min_length": 116.5,
|
||
|
|
"eval_completions/min_terminated_length": 116.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 838663420.0,
|
||
|
|
"eval_reward": 1.080160140991211,
|
||
|
|
"eval_reward_std": 0.28166723251342773,
|
||
|
|
"eval_rewards/accuracy_reward": 0.3671875,
|
||
|
|
"eval_rewards/brier_reward": 0.8043310642242432,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.47900390625,
|
||
|
|
"eval_rewards/smce": -0.11199219152331352,
|
||
|
|
"eval_runtime": 12.7377,
|
||
|
|
"eval_samples_per_second": 39.253,
|
||
|
|
"eval_steps_per_second": 0.157,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 752.4,
|
||
|
|
"completions/max_terminated_length": 534.4,
|
||
|
|
"completions/mean_length": 197.24560546875,
|
||
|
|
"completions/mean_terminated_length": 197.114404296875,
|
||
|
|
"completions/min_length": 102.2,
|
||
|
|
"completions/min_terminated_length": 102.2,
|
||
|
|
"epoch": 0.816,
|
||
|
|
"grad_norm": 0.0008329146658070385,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 855782383.0,
|
||
|
|
"reward": 1.1449370622634887,
|
||
|
|
"reward_std": 0.11253967136144638,
|
||
|
|
"rewards/accuracy_reward": 0.48984375,
|
||
|
|
"rewards/brier_reward": 0.802400529384613,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5037607312202453,
|
||
|
|
"rewards/smce": -0.022733921930193902,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 886.8,
|
||
|
|
"completions/max_terminated_length": 460.2,
|
||
|
|
"completions/mean_length": 204.29169921875,
|
||
|
|
"completions/mean_terminated_length": 204.0310089111328,
|
||
|
|
"completions/min_length": 104.4,
|
||
|
|
"completions/min_terminated_length": 104.4,
|
||
|
|
"epoch": 0.832,
|
||
|
|
"grad_norm": 0.0008481323602609336,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 872882682.0,
|
||
|
|
"reward": 1.150204586982727,
|
||
|
|
"reward_std": 0.09409180730581283,
|
||
|
|
"rewards/accuracy_reward": 0.501171875,
|
||
|
|
"rewards/brier_reward": 0.8049479007720948,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4569570302963257,
|
||
|
|
"rewards/smce": -0.05516120456159115,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1117.8,
|
||
|
|
"completions/max_terminated_length": 500.0,
|
||
|
|
"completions/mean_length": 202.97705078125,
|
||
|
|
"completions/mean_terminated_length": 202.45654907226563,
|
||
|
|
"completions/min_length": 105.0,
|
||
|
|
"completions/min_terminated_length": 105.0,
|
||
|
|
"epoch": 0.848,
|
||
|
|
"grad_norm": 0.0008220033487305045,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 889975535.0,
|
||
|
|
"reward": 1.1360312700271606,
|
||
|
|
"reward_std": 0.09862071722745895,
|
||
|
|
"rewards/accuracy_reward": 0.4732421875,
|
||
|
|
"rewards/brier_reward": 0.8020096898078919,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4578730583190918,
|
||
|
|
"rewards/smce": -0.027996178716421127,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 975.6,
|
||
|
|
"completions/max_terminated_length": 603.2,
|
||
|
|
"completions/mean_length": 204.6900390625,
|
||
|
|
"completions/mean_terminated_length": 204.43079223632813,
|
||
|
|
"completions/min_length": 97.6,
|
||
|
|
"completions/min_terminated_length": 97.6,
|
||
|
|
"epoch": 0.864,
|
||
|
|
"grad_norm": 0.0009190791170112789,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 907058377.0,
|
||
|
|
"reward": 1.154773187637329,
|
||
|
|
"reward_std": 0.1105512484908104,
|
||
|
|
"rewards/accuracy_reward": 0.512109375,
|
||
|
|
"rewards/brier_reward": 0.8000603199005127,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5104794800281525,
|
||
|
|
"rewards/smce": -0.02233685217797756,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 801.8,
|
||
|
|
"completions/max_terminated_length": 598.8,
|
||
|
|
"completions/mean_length": 201.2875,
|
||
|
|
"completions/mean_terminated_length": 201.1578857421875,
|
||
|
|
"completions/min_length": 99.0,
|
||
|
|
"completions/min_terminated_length": 99.0,
|
||
|
|
"epoch": 0.88,
|
||
|
|
"grad_norm": 0.0007379170856438577,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 924266633.0,
|
||
|
|
"reward": 1.1296437740325929,
|
||
|
|
"reward_std": 0.10275594592094421,
|
||
|
|
"rewards/accuracy_reward": 0.460546875,
|
||
|
|
"rewards/brier_reward": 0.8046643137931824,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5026220738887787,
|
||
|
|
"rewards/smce": -0.05729248225688934,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 486.6,
|
||
|
|
"completions/max_terminated_length": 486.6,
|
||
|
|
"completions/mean_length": 199.25869140625,
|
||
|
|
"completions/mean_terminated_length": 199.25869140625,
|
||
|
|
"completions/min_length": 100.6,
|
||
|
|
"completions/min_terminated_length": 100.6,
|
||
|
|
"epoch": 0.896,
|
||
|
|
"grad_norm": 0.0009705667616799474,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 941417890.0,
|
||
|
|
"reward": 1.1350660562515258,
|
||
|
|
"reward_std": 0.09697622209787368,
|
||
|
|
"rewards/accuracy_reward": 0.47099609375,
|
||
|
|
"rewards/brier_reward": 0.804245901107788,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.47399707436561583,
|
||
|
|
"rewards/smce": -0.049154537916183474,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 692.0,
|
||
|
|
"completions/max_terminated_length": 475.6,
|
||
|
|
"completions/mean_length": 201.79638671875,
|
||
|
|
"completions/mean_terminated_length": 201.66575927734374,
|
||
|
|
"completions/min_length": 94.8,
|
||
|
|
"completions/min_terminated_length": 94.8,
|
||
|
|
"epoch": 0.912,
|
||
|
|
"grad_norm": 0.0009069403749890625,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 958535581.0,
|
||
|
|
"reward": 1.1326754093170166,
|
||
|
|
"reward_std": 0.09280429035425186,
|
||
|
|
"rewards/accuracy_reward": 0.466796875,
|
||
|
|
"rewards/brier_reward": 0.8028522849082946,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4540849685668945,
|
||
|
|
"rewards/smce": -0.04201528280973434,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 868.4,
|
||
|
|
"completions/max_terminated_length": 445.2,
|
||
|
|
"completions/mean_length": 200.70908203125,
|
||
|
|
"completions/mean_terminated_length": 200.4482849121094,
|
||
|
|
"completions/min_length": 102.4,
|
||
|
|
"completions/min_terminated_length": 102.4,
|
||
|
|
"epoch": 0.928,
|
||
|
|
"grad_norm": 0.0009814233053475618,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 975617658.0,
|
||
|
|
"reward": 1.143230175971985,
|
||
|
|
"reward_std": 0.09210614264011383,
|
||
|
|
"rewards/accuracy_reward": 0.4984375,
|
||
|
|
"rewards/brier_reward": 0.7931683301925659,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.47999219298362733,
|
||
|
|
"rewards/smce": -0.04950987063348293,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1091.2,
|
||
|
|
"completions/max_terminated_length": 500.2,
|
||
|
|
"completions/mean_length": 200.055859375,
|
||
|
|
"completions/mean_terminated_length": 199.66469116210936,
|
||
|
|
"completions/min_length": 98.0,
|
||
|
|
"completions/min_terminated_length": 98.0,
|
||
|
|
"epoch": 0.944,
|
||
|
|
"grad_norm": 0.0007579100783914328,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 992641654.0,
|
||
|
|
"reward": 1.1395776748657227,
|
||
|
|
"reward_std": 0.1035296618938446,
|
||
|
|
"rewards/accuracy_reward": 0.4876953125,
|
||
|
|
"rewards/brier_reward": 0.7984759330749511,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4942158281803131,
|
||
|
|
"rewards/smce": -0.06723861880600453,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 755.6,
|
||
|
|
"completions/max_terminated_length": 591.6,
|
||
|
|
"completions/mean_length": 203.0103515625,
|
||
|
|
"completions/mean_terminated_length": 202.48793029785156,
|
||
|
|
"completions/min_length": 103.2,
|
||
|
|
"completions/min_terminated_length": 103.2,
|
||
|
|
"epoch": 0.96,
|
||
|
|
"grad_norm": 0.0008992342045530677,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 1009660800.0,
|
||
|
|
"reward": 1.1296122550964356,
|
||
|
|
"reward_std": 0.09960427284240722,
|
||
|
|
"rewards/accuracy_reward": 0.46044921875,
|
||
|
|
"rewards/brier_reward": 0.8044404029846192,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5082920014858245,
|
||
|
|
"rewards/smce": -0.05275399778038263,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.96,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 347.5,
|
||
|
|
"eval_completions/max_terminated_length": 347.5,
|
||
|
|
"eval_completions/mean_length": 202.71531677246094,
|
||
|
|
"eval_completions/mean_terminated_length": 202.71531677246094,
|
||
|
|
"eval_completions/min_length": 107.0,
|
||
|
|
"eval_completions/min_terminated_length": 107.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 1009660800.0,
|
||
|
|
"eval_reward": 1.1001547574996948,
|
||
|
|
"eval_reward_std": 0.2800827920436859,
|
||
|
|
"eval_rewards/accuracy_reward": 0.39453125,
|
||
|
|
"eval_rewards/brier_reward": 0.8129736185073853,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4657226651906967,
|
||
|
|
"eval_rewards/smce": -0.07196288928389549,
|
||
|
|
"eval_runtime": 12.214,
|
||
|
|
"eval_samples_per_second": 40.937,
|
||
|
|
"eval_steps_per_second": 0.164,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 902.2,
|
||
|
|
"completions/max_terminated_length": 543.6,
|
||
|
|
"completions/mean_length": 201.16455078125,
|
||
|
|
"completions/mean_terminated_length": 200.51107788085938,
|
||
|
|
"completions/min_length": 103.2,
|
||
|
|
"completions/min_terminated_length": 103.2,
|
||
|
|
"epoch": 0.976,
|
||
|
|
"grad_norm": 0.0007909019477665424,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"num_tokens": 1026581845.0,
|
||
|
|
"reward": 1.1536278963088988,
|
||
|
|
"reward_std": 0.0806884527206421,
|
||
|
|
"rewards/accuracy_reward": 0.5001953125,
|
||
|
|
"rewards/brier_reward": 0.8116249918937684,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.46386427283287046,
|
||
|
|
"rewards/smce": -0.04077114947140217,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 559.6,
|
||
|
|
"completions/max_terminated_length": 559.6,
|
||
|
|
"completions/mean_length": 195.6953125,
|
||
|
|
"completions/mean_terminated_length": 195.6953125,
|
||
|
|
"completions/min_length": 102.2,
|
||
|
|
"completions/min_terminated_length": 102.2,
|
||
|
|
"epoch": 0.992,
|
||
|
|
"grad_norm": 0.0007479905034415424,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 1043714245.0,
|
||
|
|
"reward": 1.1366238832473754,
|
||
|
|
"reward_std": 0.09564611464738845,
|
||
|
|
"rewards/accuracy_reward": 0.480859375,
|
||
|
|
"rewards/brier_reward": 0.7965283155441284,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4687353610992432,
|
||
|
|
"rewards/smce": -0.04140814319252968,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 442.0,
|
||
|
|
"completions/max_terminated_length": 442.0,
|
||
|
|
"completions/mean_length": 192.87665557861328,
|
||
|
|
"completions/mean_terminated_length": 192.87665557861328,
|
||
|
|
"completions/min_length": 100.5,
|
||
|
|
"completions/min_terminated_length": 100.5,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"num_tokens": 1050511528.0,
|
||
|
|
"reward": 1.135094702243805,
|
||
|
|
"reward_std": 0.10495336353778839,
|
||
|
|
"rewards/accuracy_reward": 0.48291015625,
|
||
|
|
"rewards/brier_reward": 0.7914325296878815,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.5201000869274139,
|
||
|
|
"rewards/smce": -0.04154240898787975,
|
||
|
|
"step": 312,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.00407086915840456,
|
||
|
|
"train_runtime": 42825.4883,
|
||
|
|
"train_samples_per_second": 0.467,
|
||
|
|
"train_steps_per_second": 0.007
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 312,
|
||
|
|
"num_input_tokens_seen": 1050511528,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 60,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|