84af7a58f0
Model: cjiao/goldengoose-divsweep_goose_n512_grouporc_tau1.00-7grp Source: Original Platform
1385 lines
51 KiB
JSON
1385 lines
51 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.11160714285714286,
|
|
"eval_steps": 500,
|
|
"global_step": 50,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1639.0,
|
|
"completions/max_terminated_length": 1639.0,
|
|
"completions/mean_length": 609.15625,
|
|
"completions/mean_terminated_length": 609.15625,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"epoch": 0.002232142857142857,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.2023043632507324,
|
|
"learning_rate": 0.0,
|
|
"loss": -0.0,
|
|
"num_tokens": 126514.0,
|
|
"reward": 0.3804687261581421,
|
|
"reward_std": 0.3518281579017639,
|
|
"rewards/format_reward/mean": 0.3671875,
|
|
"rewards/format_reward/std": 0.22257846593856812,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1383.0,
|
|
"completions/mean_length": 572.40625,
|
|
"completions/mean_terminated_length": 524.8064575195312,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.004464285714285714,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 25.689533233642578,
|
|
"learning_rate": 2e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 260700.0,
|
|
"reward": 0.21171873807907104,
|
|
"reward_std": 0.3009757697582245,
|
|
"rewards/format_reward/mean": 0.3984375,
|
|
"rewards/format_reward/std": 0.22146137058734894,
|
|
"rewards/mcq_exact_match_reward/mean": 0.171875,
|
|
"rewards/mcq_exact_match_reward/std": 0.38025420904159546,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1475.0,
|
|
"completions/max_terminated_length": 1475.0,
|
|
"completions/mean_length": 496.328125,
|
|
"completions/mean_terminated_length": 496.328125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"epoch": 0.006696428571428571,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.14404582977295,
|
|
"learning_rate": 4e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 366361.0,
|
|
"reward": 0.2242187261581421,
|
|
"reward_std": 0.3805117607116699,
|
|
"rewards/format_reward/mean": 0.3671875,
|
|
"rewards/format_reward/std": 0.23974503576755524,
|
|
"rewards/mcq_exact_match_reward/mean": 0.1875,
|
|
"rewards/mcq_exact_match_reward/std": 0.39339789748191833,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1414.0,
|
|
"completions/max_terminated_length": 1414.0,
|
|
"completions/mean_length": 349.609375,
|
|
"completions/mean_terminated_length": 349.609375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"epoch": 0.008928571428571428,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 37.033538818359375,
|
|
"learning_rate": 6e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 469360.0,
|
|
"reward": 0.38359373807907104,
|
|
"reward_std": 0.3727972209453583,
|
|
"rewards/format_reward/mean": 0.3984375,
|
|
"rewards/format_reward/std": 0.25479042530059814,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1515.0,
|
|
"completions/mean_length": 654.1875,
|
|
"completions/mean_terminated_length": 585.6392822265625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.011160714285714286,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.6391189098358154,
|
|
"learning_rate": 8e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 606908.0,
|
|
"reward": 0.2398437261581421,
|
|
"reward_std": 0.3410371243953705,
|
|
"rewards/format_reward/mean": 0.3671875,
|
|
"rewards/format_reward/std": 0.23974503576755524,
|
|
"rewards/mcq_exact_match_reward/mean": 0.203125,
|
|
"rewards/mcq_exact_match_reward/std": 0.40550529956817627,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2025.0,
|
|
"completions/max_terminated_length": 2025.0,
|
|
"completions/mean_length": 505.546875,
|
|
"completions/mean_terminated_length": 505.546875,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"epoch": 0.013392857142857142,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 3.158848524093628,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 722519.0,
|
|
"reward": 0.34765625,
|
|
"reward_std": 0.35713839530944824,
|
|
"rewards/format_reward/mean": 0.3515625,
|
|
"rewards/format_reward/std": 0.26246222853660583,
|
|
"rewards/mcq_exact_match_reward/mean": 0.3125,
|
|
"rewards/mcq_exact_match_reward/std": 0.467176616191864,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1934.0,
|
|
"completions/max_terminated_length": 1934.0,
|
|
"completions/mean_length": 470.03125,
|
|
"completions/mean_terminated_length": 470.03125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.015625,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 13.845512390136719,
|
|
"learning_rate": 9.99726628670463e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 845529.0,
|
|
"reward": 0.4609374701976776,
|
|
"reward_std": 0.43220406770706177,
|
|
"rewards/format_reward/mean": 0.390625,
|
|
"rewards/format_reward/std": 0.2592533528804779,
|
|
"rewards/mcq_exact_match_reward/mean": 0.421875,
|
|
"rewards/mcq_exact_match_reward/std": 0.49776285886764526,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1473.0,
|
|
"completions/mean_length": 453.65625,
|
|
"completions/mean_terminated_length": 428.3492431640625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.017857142857142856,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 22.4105281829834,
|
|
"learning_rate": 9.989068136093872e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 950339.0,
|
|
"reward": 0.24531249701976776,
|
|
"reward_std": 0.321063756942749,
|
|
"rewards/format_reward/mean": 0.421875,
|
|
"rewards/format_reward/std": 0.3238992393016815,
|
|
"rewards/mcq_exact_match_reward/mean": 0.203125,
|
|
"rewards/mcq_exact_match_reward/std": 0.40550529956817627,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1569.0,
|
|
"completions/mean_length": 697.828125,
|
|
"completions/mean_terminated_length": 654.274169921875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.020089285714285716,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.999773979187012,
|
|
"learning_rate": 9.975414512725056e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1106384.0,
|
|
"reward": 0.2109374850988388,
|
|
"reward_std": 0.27513793110847473,
|
|
"rewards/format_reward/mean": 0.390625,
|
|
"rewards/format_reward/std": 0.24346621334552765,
|
|
"rewards/mcq_exact_match_reward/mean": 0.171875,
|
|
"rewards/mcq_exact_match_reward/std": 0.38025420904159546,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1300.0,
|
|
"completions/max_terminated_length": 1300.0,
|
|
"completions/mean_length": 368.015625,
|
|
"completions/mean_terminated_length": 368.015625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.022321428571428572,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 29.1524600982666,
|
|
"learning_rate": 9.956320346634875e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1224897.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.30659258365631104,
|
|
"rewards/format_reward/mean": 0.390625,
|
|
"rewards/format_reward/std": 0.2592533528804779,
|
|
"rewards/mcq_exact_match_reward/mean": 0.234375,
|
|
"rewards/mcq_exact_match_reward/std": 0.42695629596710205,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1506.0,
|
|
"completions/mean_length": 439.46875,
|
|
"completions/mean_terminated_length": 387.58062744140625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.024553571428571428,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.193141460418701,
|
|
"learning_rate": 9.931806517013612e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 1337247.0,
|
|
"reward": 0.39531245827674866,
|
|
"reward_std": 0.3140166997909546,
|
|
"rewards/format_reward/mean": 0.359375,
|
|
"rewards/format_reward/std": 0.22658175230026245,
|
|
"rewards/mcq_exact_match_reward/mean": 0.359375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4836103618144989,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1578.0,
|
|
"completions/max_terminated_length": 1578.0,
|
|
"completions/mean_length": 526.609375,
|
|
"completions/mean_terminated_length": 526.609375,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"epoch": 0.026785714285714284,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.610779762268066,
|
|
"learning_rate": 9.901899829374047e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 1471670.0,
|
|
"reward": 0.3179687261581421,
|
|
"reward_std": 0.38177454471588135,
|
|
"rewards/format_reward/mean": 0.3671875,
|
|
"rewards/format_reward/std": 0.22257846593856812,
|
|
"rewards/mcq_exact_match_reward/mean": 0.28125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4531635046005249,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1653.0,
|
|
"completions/mean_length": 558.546875,
|
|
"completions/mean_terminated_length": 534.90478515625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.029017857142857144,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6152477264404297,
|
|
"learning_rate": 9.866632986240029e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1588257.0,
|
|
"reward": 0.24453124403953552,
|
|
"reward_std": 0.3110152781009674,
|
|
"rewards/format_reward/mean": 0.4140625,
|
|
"rewards/format_reward/std": 0.27537402510643005,
|
|
"rewards/mcq_exact_match_reward/mean": 0.203125,
|
|
"rewards/mcq_exact_match_reward/std": 0.40550529956817627,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1594.0,
|
|
"completions/max_terminated_length": 1594.0,
|
|
"completions/mean_length": 475.703125,
|
|
"completions/mean_terminated_length": 475.703125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.03125,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 4.853925704956055,
|
|
"learning_rate": 9.826044551386742e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1720934.0,
|
|
"reward": 0.22890622913837433,
|
|
"reward_std": 0.2894315719604492,
|
|
"rewards/format_reward/mean": 0.4140625,
|
|
"rewards/format_reward/std": 0.2605654299259186,
|
|
"rewards/mcq_exact_match_reward/mean": 0.1875,
|
|
"rewards/mcq_exact_match_reward/std": 0.39339789748191833,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1740.0,
|
|
"completions/mean_length": 696.109375,
|
|
"completions/mean_terminated_length": 629.6229248046875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.033482142857142856,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.440049171447754,
|
|
"learning_rate": 9.780178907671788e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 1862341.0,
|
|
"reward": 0.39531248807907104,
|
|
"reward_std": 0.3890523314476013,
|
|
"rewards/format_reward/mean": 0.359375,
|
|
"rewards/format_reward/std": 0.22658175230026245,
|
|
"rewards/mcq_exact_match_reward/mean": 0.359375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4836103618144989,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1724.0,
|
|
"completions/mean_length": 585.90625,
|
|
"completions/mean_terminated_length": 562.6984252929688,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.03571428571428571,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.889235496520996,
|
|
"learning_rate": 9.729086208503173e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1991975.0,
|
|
"reward": 0.3343749940395355,
|
|
"reward_std": 0.3780812621116638,
|
|
"rewards/format_reward/mean": 0.375,
|
|
"rewards/format_reward/std": 0.26726123690605164,
|
|
"rewards/mcq_exact_match_reward/mean": 0.296875,
|
|
"rewards/mcq_exact_match_reward/std": 0.4604927599430084,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1438.0,
|
|
"completions/mean_length": 771.46875,
|
|
"completions/mean_terminated_length": 730.290283203125,
|
|
"completions/min_length": 293.0,
|
|
"completions/min_terminated_length": 293.0,
|
|
"epoch": 0.03794642857142857,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.7261731028556824,
|
|
"learning_rate": 9.672822322997304e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 2142765.0,
|
|
"reward": 0.18203124403953552,
|
|
"reward_std": 0.26640385389328003,
|
|
"rewards/format_reward/mean": 0.4140625,
|
|
"rewards/format_reward/std": 0.2280818521976471,
|
|
"rewards/mcq_exact_match_reward/mean": 0.140625,
|
|
"rewards/mcq_exact_match_reward/std": 0.3503824472427368,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1903.0,
|
|
"completions/max_terminated_length": 1903.0,
|
|
"completions/mean_length": 623.953125,
|
|
"completions/mean_terminated_length": 623.953125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.04017857142857143,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.646529197692871,
|
|
"learning_rate": 9.611448774886923e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2280298.0,
|
|
"reward": 0.23203124105930328,
|
|
"reward_std": 0.29715561866760254,
|
|
"rewards/format_reward/mean": 0.4453125,
|
|
"rewards/format_reward/std": 0.15728822350502014,
|
|
"rewards/mcq_exact_match_reward/mean": 0.1875,
|
|
"rewards/mcq_exact_match_reward/std": 0.39339789748191833,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1689.0,
|
|
"completions/mean_length": 585.21875,
|
|
"completions/mean_terminated_length": 562.0000610351562,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.04241071428571429,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.6680755615234375,
|
|
"learning_rate": 9.545032675245813e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2429904.0,
|
|
"reward": 0.4281249940395355,
|
|
"reward_std": 0.40475884079933167,
|
|
"rewards/format_reward/mean": 0.53125,
|
|
"rewards/format_reward/std": 0.17536810040473938,
|
|
"rewards/mcq_exact_match_reward/mean": 0.375,
|
|
"rewards/mcq_exact_match_reward/std": 0.48795005679130554,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1988.0,
|
|
"completions/mean_length": 737.34375,
|
|
"completions/mean_terminated_length": 672.8851928710938,
|
|
"completions/min_length": 55.0,
|
|
"completions/min_terminated_length": 55.0,
|
|
"epoch": 0.044642857142857144,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.117641806602478,
|
|
"learning_rate": 9.473646649103817e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 2581766.0,
|
|
"reward": 0.4140625,
|
|
"reward_std": 0.42353588342666626,
|
|
"rewards/format_reward/mean": 0.390625,
|
|
"rewards/format_reward/std": 0.2741328477859497,
|
|
"rewards/mcq_exact_match_reward/mean": 0.375,
|
|
"rewards/mcq_exact_match_reward/std": 0.48795005679130554,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 2032.0,
|
|
"completions/mean_length": 682.84375,
|
|
"completions/mean_terminated_length": 661.1746215820312,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.046875,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.76787281036377,
|
|
"learning_rate": 9.397368756032444e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2745580.0,
|
|
"reward": 0.3460937440395355,
|
|
"reward_std": 0.3644231855869293,
|
|
"rewards/format_reward/mean": 0.4921875,
|
|
"rewards/format_reward/std": 0.16648055613040924,
|
|
"rewards/mcq_exact_match_reward/mean": 0.296875,
|
|
"rewards/mcq_exact_match_reward/std": 0.4604927599430084,
|
|
"step": 21
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1910.0,
|
|
"completions/max_terminated_length": 1910.0,
|
|
"completions/mean_length": 636.734375,
|
|
"completions/mean_terminated_length": 636.734375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.049107142857142856,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.8484814167022705,
|
|
"learning_rate": 9.316282404787869e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2883659.0,
|
|
"reward": 0.3906249701976776,
|
|
"reward_std": 0.42049989104270935,
|
|
"rewards/format_reward/mean": 0.46875,
|
|
"rewards/format_reward/std": 0.25,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 22
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 2016.0,
|
|
"completions/mean_length": 648.96875,
|
|
"completions/mean_terminated_length": 580.1638793945312,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.05133928571428571,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 13.19649887084961,
|
|
"learning_rate": 9.230476262104676e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3047465.0,
|
|
"reward": 0.11484374850988388,
|
|
"reward_std": 0.23332995176315308,
|
|
"rewards/format_reward/mean": 0.3671875,
|
|
"rewards/format_reward/std": 0.22257846593856812,
|
|
"rewards/mcq_exact_match_reward/mean": 0.078125,
|
|
"rewards/mcq_exact_match_reward/std": 0.27048972249031067,
|
|
"step": 23
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1814.0,
|
|
"completions/mean_length": 641.75,
|
|
"completions/mean_terminated_length": 596.3870849609375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.05357142857142857,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 10.830889701843262,
|
|
"learning_rate": 9.1400441557401e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3210817.0,
|
|
"reward": 0.21328124403953552,
|
|
"reward_std": 0.2633717656135559,
|
|
"rewards/format_reward/mean": 0.4140625,
|
|
"rewards/format_reward/std": 0.30282649397850037,
|
|
"rewards/mcq_exact_match_reward/mean": 0.171875,
|
|
"rewards/mcq_exact_match_reward/std": 0.38025420904159546,
|
|
"step": 24
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1800.0,
|
|
"completions/mean_length": 710.109375,
|
|
"completions/mean_terminated_length": 688.873046875,
|
|
"completions/min_length": 221.0,
|
|
"completions/min_terminated_length": 221.0,
|
|
"epoch": 0.05580357142857143,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.8190599083900452,
|
|
"learning_rate": 9.045084971874737e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 3349960.0,
|
|
"reward": 0.3906249701976776,
|
|
"reward_std": 0.348865270614624,
|
|
"rewards/format_reward/mean": 0.46875,
|
|
"rewards/format_reward/std": 0.17536810040473938,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1323.0,
|
|
"completions/max_terminated_length": 1323.0,
|
|
"completions/mean_length": 530.28125,
|
|
"completions/mean_terminated_length": 530.28125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.05803571428571429,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 2.7582437992095947,
|
|
"learning_rate": 8.945702546981968e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3480610.0,
|
|
"reward": 0.3960937559604645,
|
|
"reward_std": 0.3427093029022217,
|
|
"rewards/format_reward/mean": 0.5234375,
|
|
"rewards/format_reward/std": 0.24282869696617126,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 26
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1864.0,
|
|
"completions/mean_length": 717.171875,
|
|
"completions/mean_terminated_length": 674.241943359375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.060267857142857144,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3076305389404297,
|
|
"learning_rate": 8.842005554284295e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3663341.0,
|
|
"reward": 0.2437499761581421,
|
|
"reward_std": 0.3142412304878235,
|
|
"rewards/format_reward/mean": 0.40625,
|
|
"rewards/format_reward/std": 0.21593283116817474,
|
|
"rewards/mcq_exact_match_reward/mean": 0.203125,
|
|
"rewards/mcq_exact_match_reward/std": 0.40550529956817627,
|
|
"step": 27
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1616.0,
|
|
"completions/mean_length": 695.453125,
|
|
"completions/mean_terminated_length": 628.9343872070312,
|
|
"completions/min_length": 215.0,
|
|
"completions/min_terminated_length": 215.0,
|
|
"epoch": 0.0625,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7634695172309875,
|
|
"learning_rate": 8.734107384920769e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 3812410.0,
|
|
"reward": 0.2945312261581421,
|
|
"reward_std": 0.32147979736328125,
|
|
"rewards/format_reward/mean": 0.4453125,
|
|
"rewards/format_reward/std": 0.2538151443004608,
|
|
"rewards/mcq_exact_match_reward/mean": 0.25,
|
|
"rewards/mcq_exact_match_reward/std": 0.4364357888698578,
|
|
"step": 28
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1797.0,
|
|
"completions/mean_length": 509.3125,
|
|
"completions/mean_terminated_length": 484.888916015625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.06473214285714286,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 6.607220649719238,
|
|
"learning_rate": 8.622126023955445e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 3937998.0,
|
|
"reward": 0.47109371423721313,
|
|
"reward_std": 0.4023832082748413,
|
|
"rewards/format_reward/mean": 0.4921875,
|
|
"rewards/format_reward/std": 0.16648055613040924,
|
|
"rewards/mcq_exact_match_reward/mean": 0.421875,
|
|
"rewards/mcq_exact_match_reward/std": 0.49776285886764526,
|
|
"step": 29
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1948.0,
|
|
"completions/max_terminated_length": 1948.0,
|
|
"completions/mean_length": 672.265625,
|
|
"completions/mean_terminated_length": 672.265625,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"epoch": 0.06696428571428571,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 1.1837975978851318,
|
|
"learning_rate": 8.506183921362442e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4069807.0,
|
|
"reward": 0.42265623807907104,
|
|
"reward_std": 0.24412530660629272,
|
|
"rewards/format_reward/mean": 0.4765625,
|
|
"rewards/format_reward/std": 0.28770697116851807,
|
|
"rewards/mcq_exact_match_reward/mean": 0.375,
|
|
"rewards/mcq_exact_match_reward/std": 0.48795005679130554,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1799.0,
|
|
"completions/mean_length": 536.515625,
|
|
"completions/mean_terminated_length": 512.5238647460938,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.06919642857142858,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 14.385038375854492,
|
|
"learning_rate": 8.386407858128706e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4179424.0,
|
|
"reward": 0.3453125059604645,
|
|
"reward_std": 0.3491568863391876,
|
|
"rewards/format_reward/mean": 0.484375,
|
|
"rewards/format_reward/std": 0.23517554998397827,
|
|
"rewards/mcq_exact_match_reward/mean": 0.296875,
|
|
"rewards/mcq_exact_match_reward/std": 0.4604927599430084,
|
|
"step": 31
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1616.0,
|
|
"completions/mean_length": 537.859375,
|
|
"completions/mean_terminated_length": 513.888916015625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.07142857142857142,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 10.334357261657715,
|
|
"learning_rate": 8.262928807620843e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4308543.0,
|
|
"reward": 0.3101562261581421,
|
|
"reward_std": 0.37167930603027344,
|
|
"rewards/format_reward/mean": 0.4453125,
|
|
"rewards/format_reward/std": 0.15728822350502014,
|
|
"rewards/mcq_exact_match_reward/mean": 0.265625,
|
|
"rewards/mcq_exact_match_reward/std": 0.44515693187713623,
|
|
"step": 32
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1829.0,
|
|
"completions/mean_length": 727.734375,
|
|
"completions/mean_terminated_length": 685.1451416015625,
|
|
"completions/min_length": 207.0,
|
|
"completions/min_terminated_length": 207.0,
|
|
"epoch": 0.07366071428571429,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.7631356120109558,
|
|
"learning_rate": 8.135881792367685e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4472694.0,
|
|
"reward": 0.31953126192092896,
|
|
"reward_std": 0.35187065601348877,
|
|
"rewards/format_reward/mean": 0.5390625,
|
|
"rewards/format_reward/std": 0.20518454909324646,
|
|
"rewards/mcq_exact_match_reward/mean": 0.265625,
|
|
"rewards/mcq_exact_match_reward/std": 0.44515693187713623,
|
|
"step": 33
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1818.0,
|
|
"completions/mean_length": 534.390625,
|
|
"completions/mean_terminated_length": 485.56451416015625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.07589285714285714,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 5.339401721954346,
|
|
"learning_rate": 8.005405736415125e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4603871.0,
|
|
"reward": 0.4007812738418579,
|
|
"reward_std": 0.26095426082611084,
|
|
"rewards/format_reward/mean": 0.5703125,
|
|
"rewards/format_reward/std": 0.26528194546699524,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 34
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1676.0,
|
|
"completions/mean_length": 726.609375,
|
|
"completions/mean_terminated_length": 589.913818359375,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"epoch": 0.078125,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0368614196777344,
|
|
"learning_rate": 7.871643313414718e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4731646.0,
|
|
"reward": 0.3101562261581421,
|
|
"reward_std": 0.3630598783493042,
|
|
"rewards/format_reward/mean": 0.4453125,
|
|
"rewards/format_reward/std": 0.20152568817138672,
|
|
"rewards/mcq_exact_match_reward/mean": 0.265625,
|
|
"rewards/mcq_exact_match_reward/std": 0.44515693187713623,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1353.0,
|
|
"completions/mean_length": 511.03125,
|
|
"completions/mean_terminated_length": 486.63494873046875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.08035714285714286,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 10.782816886901855,
|
|
"learning_rate": 7.734740790612136e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 4839608.0,
|
|
"reward": 0.22343748807907104,
|
|
"reward_std": 0.3053799867630005,
|
|
"rewards/format_reward/mean": 0.515625,
|
|
"rewards/format_reward/std": 0.17747680842876434,
|
|
"rewards/mcq_exact_match_reward/mean": 0.171875,
|
|
"rewards/mcq_exact_match_reward/std": 0.38025420904159546,
|
|
"step": 36
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1972.0,
|
|
"completions/mean_length": 674.5625,
|
|
"completions/mean_terminated_length": 652.761962890625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.08258928571428571,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.231216907501221,
|
|
"learning_rate": 7.594847868906076e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4977508.0,
|
|
"reward": 0.23671874403953552,
|
|
"reward_std": 0.35034993290901184,
|
|
"rewards/format_reward/mean": 0.4921875,
|
|
"rewards/format_reward/std": 0.2744719088077545,
|
|
"rewards/mcq_exact_match_reward/mean": 0.1875,
|
|
"rewards/mcq_exact_match_reward/std": 0.39339789748191833,
|
|
"step": 37
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.078125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1850.0,
|
|
"completions/mean_length": 762.984375,
|
|
"completions/mean_terminated_length": 654.084716796875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.08482142857142858,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 3.4671003818511963,
|
|
"learning_rate": 7.452117519152541e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 5138179.0,
|
|
"reward": 0.2828124761581421,
|
|
"reward_std": 0.387776255607605,
|
|
"rewards/format_reward/mean": 0.484375,
|
|
"rewards/format_reward/std": 0.2812775671482086,
|
|
"rewards/mcq_exact_match_reward/mean": 0.234375,
|
|
"rewards/mcq_exact_match_reward/std": 0.42695629596710205,
|
|
"step": 38
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1828.0,
|
|
"completions/max_terminated_length": 1828.0,
|
|
"completions/mean_length": 557.0,
|
|
"completions/mean_terminated_length": 557.0,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.08705357142857142,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.978475570678711,
|
|
"learning_rate": 7.306705814893439e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 5254915.0,
|
|
"reward": 0.5828124284744263,
|
|
"reward_std": 0.37641221284866333,
|
|
"rewards/format_reward/mean": 0.515625,
|
|
"rewards/format_reward/std": 0.1985812783241272,
|
|
"rewards/mcq_exact_match_reward/mean": 0.53125,
|
|
"rewards/mcq_exact_match_reward/std": 0.5029674172401428,
|
|
"step": 39
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1996.0,
|
|
"completions/mean_length": 549.765625,
|
|
"completions/mean_terminated_length": 525.984130859375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.08928571428571429,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 14.763313293457031,
|
|
"learning_rate": 7.158771761692464e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 5366588.0,
|
|
"reward": 0.2992187440395355,
|
|
"reward_std": 0.2617577314376831,
|
|
"rewards/format_reward/mean": 0.4921875,
|
|
"rewards/format_reward/std": 0.2744719088077545,
|
|
"rewards/mcq_exact_match_reward/mean": 0.25,
|
|
"rewards/mcq_exact_match_reward/std": 0.4364357888698578,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 2036.0,
|
|
"completions/mean_length": 695.21875,
|
|
"completions/mean_terminated_length": 628.6884765625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.09151785714285714,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.990268230438232,
|
|
"learning_rate": 7.008477123264847e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 5525970.0,
|
|
"reward": 0.3609374761581421,
|
|
"reward_std": 0.28916043043136597,
|
|
"rewards/format_reward/mean": 0.484375,
|
|
"rewards/format_reward/std": 0.2514837086200714,
|
|
"rewards/mcq_exact_match_reward/mean": 0.3125,
|
|
"rewards/mcq_exact_match_reward/std": 0.467176616191864,
|
|
"step": 41
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1187.0,
|
|
"completions/max_terminated_length": 1187.0,
|
|
"completions/mean_length": 503.046875,
|
|
"completions/mean_terminated_length": 503.046875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.09375,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.825297832489014,
|
|
"learning_rate": 6.855986244591103e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 5652813.0,
|
|
"reward": 0.3109374940395355,
|
|
"reward_std": 0.27216827869415283,
|
|
"rewards/format_reward/mean": 0.453125,
|
|
"rewards/format_reward/std": 0.23091863095760345,
|
|
"rewards/mcq_exact_match_reward/mean": 0.265625,
|
|
"rewards/mcq_exact_match_reward/std": 0.44515693187713623,
|
|
"step": 42
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1779.0,
|
|
"completions/mean_length": 698.46875,
|
|
"completions/mean_terminated_length": 632.0983276367188,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.09598214285714286,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 13.259655952453613,
|
|
"learning_rate": 6.701465872208216e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 5805515.0,
|
|
"reward": 0.4039062261581421,
|
|
"reward_std": 0.42280226945877075,
|
|
"rewards/format_reward/mean": 0.4453125,
|
|
"rewards/format_reward/std": 0.29703813791275024,
|
|
"rewards/mcq_exact_match_reward/mean": 0.359375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4836103618144989,
|
|
"step": 43
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1473.0,
|
|
"completions/mean_length": 426.15625,
|
|
"completions/mean_terminated_length": 373.83868408203125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.09821428571428571,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 17.04229164123535,
|
|
"learning_rate": 6.545084971874736e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 5926829.0,
|
|
"reward": 0.3999999761581421,
|
|
"reward_std": 0.3521348237991333,
|
|
"rewards/format_reward/mean": 0.5625,
|
|
"rewards/format_reward/std": 0.208927720785141,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 44
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1995.0,
|
|
"completions/mean_length": 559.859375,
|
|
"completions/mean_terminated_length": 511.8548278808594,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.10044642857142858,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 16.93348503112793,
|
|
"learning_rate": 6.387014543809223e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 6053444.0,
|
|
"reward": 0.40468746423721313,
|
|
"reward_std": 0.3729129731655121,
|
|
"rewards/format_reward/mean": 0.609375,
|
|
"rewards/format_reward/std": 0.24346621334552765,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1565.0,
|
|
"completions/max_terminated_length": 1565.0,
|
|
"completions/mean_length": 373.53125,
|
|
"completions/mean_terminated_length": 373.53125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.10267857142857142,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 16.590978622436523,
|
|
"learning_rate": 6.227427435703995e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 6187174.0,
|
|
"reward": 0.27421873807907104,
|
|
"reward_std": 0.33643338084220886,
|
|
"rewards/format_reward/mean": 0.5546875,
|
|
"rewards/format_reward/std": 0.20152568817138672,
|
|
"rewards/mcq_exact_match_reward/mean": 0.21875,
|
|
"rewards/mcq_exact_match_reward/std": 0.4166666865348816,
|
|
"step": 46
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1133.0,
|
|
"completions/max_terminated_length": 1133.0,
|
|
"completions/mean_length": 301.3125,
|
|
"completions/mean_terminated_length": 301.3125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.10491071428571429,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 40.43541717529297,
|
|
"learning_rate": 6.066498153718734e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 6299226.0,
|
|
"reward": 0.38515621423721313,
|
|
"reward_std": 0.33590227365493774,
|
|
"rewards/format_reward/mean": 0.5703125,
|
|
"rewards/format_reward/std": 0.19654127955436707,
|
|
"rewards/mcq_exact_match_reward/mean": 0.328125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4732423722743988,
|
|
"step": 47
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1598.0,
|
|
"completions/mean_length": 501.921875,
|
|
"completions/mean_terminated_length": 477.3809814453125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.10714285714285714,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 8.04982852935791,
|
|
"learning_rate": 5.90440267166055e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 6423469.0,
|
|
"reward": 0.40156251192092896,
|
|
"reward_std": 0.3936265707015991,
|
|
"rewards/format_reward/mean": 0.578125,
|
|
"rewards/format_reward/std": 0.25539806485176086,
|
|
"rewards/mcq_exact_match_reward/mean": 0.34375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4787135720252991,
|
|
"step": 48
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1493.0,
|
|
"completions/mean_length": 285.203125,
|
|
"completions/mean_terminated_length": 257.22222900390625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.109375,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 21.848068237304688,
|
|
"learning_rate": 5.741318238559209e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 6559442.0,
|
|
"reward": 0.46171873807907104,
|
|
"reward_std": 0.25157496333122253,
|
|
"rewards/format_reward/mean": 0.5546875,
|
|
"rewards/format_reward/std": 0.20152568817138672,
|
|
"rewards/mcq_exact_match_reward/mean": 0.40625,
|
|
"rewards/mcq_exact_match_reward/std": 0.49501484632492065,
|
|
"step": 49
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1993.0,
|
|
"completions/mean_length": 242.203125,
|
|
"completions/mean_terminated_length": 183.9516143798828,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.11160714285714286,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 18.545976638793945,
|
|
"learning_rate": 5.577423184847931e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 6663423.0,
|
|
"reward": 0.27421873807907104,
|
|
"reward_std": 0.30959123373031616,
|
|
"rewards/format_reward/mean": 0.7109375,
|
|
"rewards/format_reward/std": 0.2928335666656494,
|
|
"rewards/mcq_exact_match_reward/mean": 0.203125,
|
|
"rewards/mcq_exact_match_reward/std": 0.40550529956817627,
|
|
"step": 50
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 100,
|
|
"num_input_tokens_seen": 6663423,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|