12c1e2fb68
Model: cjiao/goldengoose-divsweep_goose_n512_indorc_tau2.00-7grp Source: Original Platform
1385 lines
50 KiB
JSON
1385 lines
50 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.11160714285714286,
|
|
"eval_steps": 500,
|
|
"global_step": 50,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 2011.0,
|
|
"completions/mean_length": 528.109375,
|
|
"completions/mean_terminated_length": 503.9841613769531,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"epoch": 0.002232142857142857,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9355294108390808,
|
|
"learning_rate": 0.0,
|
|
"loss": -0.0,
|
|
"num_tokens": 121119.0,
|
|
"reward": 0.26953125,
|
|
"reward_std": 0.37513452768325806,
|
|
"rewards/format_reward/mean": 0.3515625,
|
|
"rewards/format_reward/std": 0.27716949582099915,
|
|
"rewards/mcq_exact_match_reward/mean": 0.234375,
|
|
"rewards/mcq_exact_match_reward/std": 0.42695629596710205,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1501.0,
|
|
"completions/max_terminated_length": 1501.0,
|
|
"completions/mean_length": 427.296875,
|
|
"completions/mean_terminated_length": 427.296875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.004464285714285714,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.184840202331543,
|
|
"learning_rate": 2e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 243994.0,
|
|
"reward": 0.3226562440395355,
|
|
"reward_std": 0.42037320137023926,
|
|
"rewards/format_reward/mean": 0.4140625,
|
|
"rewards/format_reward/std": 0.2605654299259186,
|
|
"rewards/mcq_exact_match_reward/mean": 0.28125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4531635046005249,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1531.0,
|
|
"completions/mean_length": 544.0,
|
|
"completions/mean_terminated_length": 495.4838562011719,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.006696428571428571,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.455195903778076,
|
|
"learning_rate": 4e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 382882.0,
|
|
"reward": 0.15468749403953552,
|
|
"reward_std": 0.24445468187332153,
|
|
"rewards/format_reward/mean": 0.296875,
|
|
"rewards/format_reward/std": 0.2630521357059479,
|
|
"rewards/mcq_exact_match_reward/mean": 0.125,
|
|
"rewards/mcq_exact_match_reward/std": 0.3333333432674408,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1767.0,
|
|
"completions/mean_length": 446.203125,
|
|
"completions/mean_terminated_length": 367.42620849609375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"epoch": 0.008928571428571428,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 26.635173797607422,
|
|
"learning_rate": 6e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 498159.0,
|
|
"reward": 0.17031247913837433,
|
|
"reward_std": 0.2350117713212967,
|
|
"rewards/format_reward/mean": 0.296875,
|
|
"rewards/format_reward/std": 0.24750742316246033,
|
|
"rewards/mcq_exact_match_reward/mean": 0.140625,
|
|
"rewards/mcq_exact_match_reward/std": 0.3503824472427368,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1692.0,
|
|
"completions/mean_length": 401.390625,
|
|
"completions/mean_terminated_length": 348.274169921875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"epoch": 0.011160714285714286,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 22.468753814697266,
|
|
"learning_rate": 8e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 606640.0,
|
|
"reward": 0.3656249940395355,
|
|
"reward_std": 0.4346795678138733,
|
|
"rewards/format_reward/mean": 0.375,
|
|
"rewards/format_reward/std": 0.30860671401023865,
|
|
"rewards/mcq_exact_match_reward/mean": 0.328125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4732423722743988,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1654.0,
|
|
"completions/max_terminated_length": 1654.0,
|
|
"completions/mean_length": 294.734375,
|
|
"completions/mean_terminated_length": 294.734375,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"epoch": 0.013392857142857142,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.87374210357666,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 722783.0,
|
|
"reward": 0.30937498807907104,
|
|
"reward_std": 0.37140360474586487,
|
|
"rewards/format_reward/mean": 0.4375,
|
|
"rewards/format_reward/std": 0.22712838649749756,
|
|
"rewards/mcq_exact_match_reward/mean": 0.265625,
|
|
"rewards/mcq_exact_match_reward/std": 0.44515693187713623,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1824.0,
|
|
"completions/max_terminated_length": 1824.0,
|
|
"completions/mean_length": 525.296875,
|
|
"completions/mean_terminated_length": 525.296875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.015625,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.5019431114196777,
|
|
"learning_rate": 9.99726628670463e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 869410.0,
|
|
"reward": 0.06562500447034836,
|
|
"reward_std": 0.1088000163435936,
|
|
"rewards/format_reward/mean": 0.34375,
|
|
"rewards/format_reward/std": 0.3196600377559662,
|
|
"rewards/mcq_exact_match_reward/mean": 0.03125,
|
|
"rewards/mcq_exact_match_reward/std": 0.17536810040473938,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1983.0,
|
|
"completions/mean_length": 567.390625,
|
|
"completions/mean_terminated_length": 519.6290283203125,
|
|
"completions/min_length": 153.0,
|
|
"completions/min_terminated_length": 153.0,
|
|
"epoch": 0.017857142857142856,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1946548223495483,
|
|
"learning_rate": 9.989068136093872e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 1001739.0,
|
|
"reward": 0.36406245827674866,
|
|
"reward_std": 0.3973655700683594,
|
|
"rewards/format_reward/mean": 0.359375,
|
|
"rewards/format_reward/std": 0.24346621334552765,
|
|
"rewards/mcq_exact_match_reward/mean": 0.328125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4732423722743988,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1868.0,
|
|
"completions/mean_length": 313.71875,
|
|
"completions/mean_terminated_length": 286.19049072265625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"epoch": 0.020089285714285716,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 14.30042552947998,
|
|
"learning_rate": 9.975414512725056e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1119969.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.34339872002601624,
|
|
"rewards/format_reward/mean": 0.390625,
|
|
"rewards/format_reward/std": 0.2741328477859497,
|
|
"rewards/mcq_exact_match_reward/mean": 0.3125,
|
|
"rewards/mcq_exact_match_reward/std": 0.467176616191864,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1410.0,
|
|
"completions/mean_length": 439.203125,
|
|
"completions/mean_terminated_length": 387.3064270019531,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.022321428571428572,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.828161239624023,
|
|
"learning_rate": 9.956320346634875e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 1259382.0,
|
|
"reward": 0.43281248211860657,
|
|
"reward_std": 0.40298882126808167,
|
|
"rewards/format_reward/mean": 0.421875,
|
|
"rewards/format_reward/std": 0.27048972249031067,
|
|
"rewards/mcq_exact_match_reward/mean": 0.390625,
|
|
"rewards/mcq_exact_match_reward/std": 0.4917473793029785,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1549.0,
|
|
"completions/mean_length": 396.5,
|
|
"completions/mean_terminated_length": 343.2257995605469,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.024553571428571428,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 10.044999122619629,
|
|
"learning_rate": 9.931806517013612e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 1385830.0,
|
|
"reward": 0.19921875,
|
|
"reward_std": 0.276921808719635,
|
|
"rewards/format_reward/mean": 0.4296875,
|
|
"rewards/format_reward/std": 0.1751912236213684,
|
|
"rewards/mcq_exact_match_reward/mean": 0.15625,
|
|
"rewards/mcq_exact_match_reward/std": 0.36596253514289856,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1999.0,
|
|
"completions/mean_length": 445.40625,
|
|
"completions/mean_terminated_length": 419.9682922363281,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.026785714285714284,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.090798377990723,
|
|
"learning_rate": 9.901899829374047e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1481472.0,
|
|
"reward": 0.515625,
|
|
"reward_std": 0.46172386407852173,
|
|
"rewards/format_reward/mean": 0.46875,
|
|
"rewards/format_reward/std": 0.33184191584587097,
|
|
"rewards/mcq_exact_match_reward/mean": 0.46875,
|
|
"rewards/mcq_exact_match_reward/std": 0.5029674172401428,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1761.0,
|
|
"completions/mean_length": 581.34375,
|
|
"completions/mean_terminated_length": 534.0322265625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.029017857142857144,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 10.792880058288574,
|
|
"learning_rate": 9.866632986240029e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1609150.0,
|
|
"reward": 0.16874998807907104,
|
|
"reward_std": 0.27205121517181396,
|
|
"rewards/format_reward/mean": 0.4375,
|
|
"rewards/format_reward/std": 0.28867512941360474,
|
|
"rewards/mcq_exact_match_reward/mean": 0.125,
|
|
"rewards/mcq_exact_match_reward/std": 0.3333333432674408,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.046875,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 1903.0,
|
|
"completions/mean_length": 568.484375,
|
|
"completions/mean_terminated_length": 495.7212829589844,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.03125,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 8.320207595825195,
|
|
"learning_rate": 9.826044551386742e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1751637.0,
|
|
"reward": 0.4781249761581421,
|
|
"reward_std": 0.3682000935077667,
|
|
"rewards/format_reward/mean": 0.40625,
|
|
"rewards/format_reward/std": 0.233588308095932,
|
|
"rewards/mcq_exact_match_reward/mean": 0.4375,
|
|
"rewards/mcq_exact_match_reward/std": 0.5,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1356.0,
|
|
"completions/max_terminated_length": 1356.0,
|
|
"completions/mean_length": 370.96875,
|
|
"completions/mean_terminated_length": 370.96875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.033482142857142856,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 14.99644947052002,
|
|
"learning_rate": 9.780178907671788e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1877739.0,
|
|
"reward": 0.5812499523162842,
|
|
"reward_std": 0.45640966296195984,
|
|
"rewards/format_reward/mean": 0.5,
|
|
"rewards/format_reward/std": 0.26726123690605164,
|
|
"rewards/mcq_exact_match_reward/mean": 0.53125,
|
|
"rewards/mcq_exact_match_reward/std": 0.5029674172401428,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1714.0,
|
|
"completions/max_terminated_length": 1714.0,
|
|
"completions/mean_length": 428.390625,
|
|
"completions/mean_terminated_length": 428.390625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.03571428571428571,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.95833683013916,
|
|
"learning_rate": 9.729086208503173e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2002500.0,
|
|
"reward": 0.516406238079071,
|
|
"reward_std": 0.47470909357070923,
|
|
"rewards/format_reward/mean": 0.6328125,
|
|
"rewards/format_reward/std": 0.2987033724784851,
|
|
"rewards/mcq_exact_match_reward/mean": 0.453125,
|
|
"rewards/mcq_exact_match_reward/std": 0.501733124256134,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1412.0,
|
|
"completions/max_terminated_length": 1412.0,
|
|
"completions/mean_length": 255.796875,
|
|
"completions/mean_terminated_length": 255.796875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.03794642857142857,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 11.912960052490234,
|
|
"learning_rate": 9.672822322997304e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 2108527.0,
|
|
"reward": 0.49296873807907104,
|
|
"reward_std": 0.28548362851142883,
|
|
"rewards/format_reward/mean": 0.7109375,
|
|
"rewards/format_reward/std": 0.2643453776836395,
|
|
"rewards/mcq_exact_match_reward/mean": 0.421875,
|
|
"rewards/mcq_exact_match_reward/std": 0.49776285886764526,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1803.0,
|
|
"completions/max_terminated_length": 1803.0,
|
|
"completions/mean_length": 231.40625,
|
|
"completions/mean_terminated_length": 231.40625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.04017857142857143,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 20.999282836914062,
|
|
"learning_rate": 9.611448774886923e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 2223681.0,
|
|
"reward": 0.5687500238418579,
|
|
"reward_std": 0.35626089572906494,
|
|
"rewards/format_reward/mean": 0.6875,
|
|
"rewards/format_reward/std": 0.3021090030670166,
|
|
"rewards/mcq_exact_match_reward/mean": 0.5,
|
|
"rewards/mcq_exact_match_reward/std": 0.5039526224136353,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1676.0,
|
|
"completions/max_terminated_length": 1676.0,
|
|
"completions/mean_length": 235.875,
|
|
"completions/mean_terminated_length": 235.875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.04241071428571429,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 15.144044876098633,
|
|
"learning_rate": 9.545032675245813e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 2335865.0,
|
|
"reward": 0.5882812738418579,
|
|
"reward_std": 0.3023134469985962,
|
|
"rewards/format_reward/mean": 0.7265625,
|
|
"rewards/format_reward/std": 0.3077012598514557,
|
|
"rewards/mcq_exact_match_reward/mean": 0.515625,
|
|
"rewards/mcq_exact_match_reward/std": 0.5037065148353577,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 971.0,
|
|
"completions/max_terminated_length": 971.0,
|
|
"completions/mean_length": 119.5,
|
|
"completions/mean_terminated_length": 119.5,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.044642857142857144,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 14.69491195678711,
|
|
"learning_rate": 9.473646649103817e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2448625.0,
|
|
"reward": 0.37812501192092896,
|
|
"reward_std": 0.402826726436615,
|
|
"rewards/format_reward/mean": 0.8125,
|
|
"rewards/format_reward/std": 0.28867512941360474,
|
|
"rewards/mcq_exact_match_reward/mean": 0.296875,
|
|
"rewards/mcq_exact_match_reward/std": 0.4604927599430084,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1299.0,
|
|
"completions/max_terminated_length": 1299.0,
|
|
"completions/mean_length": 88.09375,
|
|
"completions/mean_terminated_length": 88.09375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.046875,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 37.22273635864258,
|
|
"learning_rate": 9.397368756032444e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 2531639.0,
|
|
"reward": 0.62109375,
|
|
"reward_std": 0.245207741856575,
|
|
"rewards/format_reward/mean": 0.8984375,
|
|
"rewards/format_reward/std": 0.22146137058734894,
|
|
"rewards/mcq_exact_match_reward/mean": 0.53125,
|
|
"rewards/mcq_exact_match_reward/std": 0.5029674172401428,
|
|
"step": 21
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 822.0,
|
|
"completions/max_terminated_length": 822.0,
|
|
"completions/mean_length": 32.828125,
|
|
"completions/mean_terminated_length": 32.828125,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.049107142857142856,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 29.922048568725586,
|
|
"learning_rate": 9.316282404787869e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2643372.0,
|
|
"reward": 0.5687500238418579,
|
|
"reward_std": 0.3287316560745239,
|
|
"rewards/format_reward/mean": 0.84375,
|
|
"rewards/format_reward/std": 0.233588308095932,
|
|
"rewards/mcq_exact_match_reward/mean": 0.484375,
|
|
"rewards/mcq_exact_match_reward/std": 0.5037065148353577,
|
|
"step": 22
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1162.0,
|
|
"completions/max_terminated_length": 1162.0,
|
|
"completions/mean_length": 81.90625,
|
|
"completions/mean_terminated_length": 81.90625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.05133928571428571,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 12.92499828338623,
|
|
"learning_rate": 9.230476262104676e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2747486.0,
|
|
"reward": 0.4671875238418579,
|
|
"reward_std": 0.3623044788837433,
|
|
"rewards/format_reward/mean": 0.921875,
|
|
"rewards/format_reward/std": 0.20351573824882507,
|
|
"rewards/mcq_exact_match_reward/mean": 0.375,
|
|
"rewards/mcq_exact_match_reward/std": 0.48795005679130554,
|
|
"step": 23
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 567.0,
|
|
"completions/max_terminated_length": 567.0,
|
|
"completions/mean_length": 41.765625,
|
|
"completions/mean_terminated_length": 41.765625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.05357142857142857,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 24.27642250061035,
|
|
"learning_rate": 9.1400441557401e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 2828591.0,
|
|
"reward": 0.39218753576278687,
|
|
"reward_std": 0.33768123388290405,
|
|
"rewards/format_reward/mean": 0.953125,
|
|
"rewards/format_reward/std": 0.14689241349697113,
|
|
"rewards/mcq_exact_match_reward/mean": 0.296875,
|
|
"rewards/mcq_exact_match_reward/std": 0.4604927599430084,
|
|
"step": 24
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 50.0,
|
|
"completions/max_terminated_length": 50.0,
|
|
"completions/mean_length": 12.109375,
|
|
"completions/mean_terminated_length": 12.109375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.05580357142857143,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 10.304224967956543,
|
|
"learning_rate": 9.045084971874737e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 2929198.0,
|
|
"reward": 0.811718761920929,
|
|
"reward_std": 0.2634497284889221,
|
|
"rewards/format_reward/mean": 0.9296875,
|
|
"rewards/format_reward/std": 0.19654127955436707,
|
|
"rewards/mcq_exact_match_reward/mean": 0.71875,
|
|
"rewards/mcq_exact_match_reward/std": 0.4531635046005249,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/max_terminated_length": 926.0,
|
|
"completions/mean_length": 68.359375,
|
|
"completions/mean_terminated_length": 68.359375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.05803571428571429,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 19.061513900756836,
|
|
"learning_rate": 8.945702546981968e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3025421.0,
|
|
"reward": 0.37109375,
|
|
"reward_std": 0.33500707149505615,
|
|
"rewards/format_reward/mean": 0.8984375,
|
|
"rewards/format_reward/std": 0.2387082874774933,
|
|
"rewards/mcq_exact_match_reward/mean": 0.28125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4531635046005249,
|
|
"step": 26
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 12.0,
|
|
"completions/max_terminated_length": 12.0,
|
|
"completions/mean_length": 12.0,
|
|
"completions/mean_terminated_length": 12.0,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.060267857142857144,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 16.958600997924805,
|
|
"learning_rate": 8.842005554284295e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3141461.0,
|
|
"reward": 0.5062500238418579,
|
|
"reward_std": 0.3335031569004059,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.40625,
|
|
"rewards/mcq_exact_match_reward/std": 0.49501484632492065,
|
|
"step": 27
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 40.0,
|
|
"completions/max_terminated_length": 40.0,
|
|
"completions/mean_length": 11.71875,
|
|
"completions/mean_terminated_length": 11.71875,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.0625,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 9.72388744354248,
|
|
"learning_rate": 8.734107384920769e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 3240051.0,
|
|
"reward": 0.38984376192092896,
|
|
"reward_std": 0.21241585910320282,
|
|
"rewards/format_reward/mean": 0.9296875,
|
|
"rewards/format_reward/std": 0.19654127955436707,
|
|
"rewards/mcq_exact_match_reward/mean": 0.296875,
|
|
"rewards/mcq_exact_match_reward/std": 0.4604927599430084,
|
|
"step": 28
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 90.0,
|
|
"completions/max_terminated_length": 90.0,
|
|
"completions/mean_length": 13.328125,
|
|
"completions/mean_terminated_length": 13.328125,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.06473214285714286,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 11.685744285583496,
|
|
"learning_rate": 8.622126023955445e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3325080.0,
|
|
"reward": 0.48906251788139343,
|
|
"reward_std": 0.22539028525352478,
|
|
"rewards/format_reward/mean": 0.984375,
|
|
"rewards/format_reward/std": 0.125,
|
|
"rewards/mcq_exact_match_reward/mean": 0.390625,
|
|
"rewards/mcq_exact_match_reward/std": 0.4917473793029785,
|
|
"step": 29
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 46.0,
|
|
"completions/max_terminated_length": 46.0,
|
|
"completions/mean_length": 14.34375,
|
|
"completions/mean_terminated_length": 14.34375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.06696428571428571,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 19.394933700561523,
|
|
"learning_rate": 8.506183921362442e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 3426318.0,
|
|
"reward": 0.45468753576278687,
|
|
"reward_std": 0.1829523742198944,
|
|
"rewards/format_reward/mean": 0.953125,
|
|
"rewards/format_reward/std": 0.21304203569889069,
|
|
"rewards/mcq_exact_match_reward/mean": 0.359375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4836103618144989,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1043.0,
|
|
"completions/max_terminated_length": 1043.0,
|
|
"completions/mean_length": 31.171875,
|
|
"completions/mean_terminated_length": 31.171875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.06919642857142858,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 10.200380325317383,
|
|
"learning_rate": 8.386407858128706e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 3505169.0,
|
|
"reward": 0.4281250238418579,
|
|
"reward_std": 0.19939783215522766,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.328125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4732423722743988,
|
|
"step": 31
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 184.0,
|
|
"completions/max_terminated_length": 184.0,
|
|
"completions/mean_length": 18.5625,
|
|
"completions/mean_terminated_length": 18.5625,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"epoch": 0.07142857142857142,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 6.094886302947998,
|
|
"learning_rate": 8.262928807620843e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 3605933.0,
|
|
"reward": 0.44687503576278687,
|
|
"reward_std": 0.1315089464187622,
|
|
"rewards/format_reward/mean": 0.875,
|
|
"rewards/format_reward/std": 0.3333333432674408,
|
|
"rewards/mcq_exact_match_reward/mean": 0.359375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4836103618144989,
|
|
"step": 32
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 143.0,
|
|
"completions/max_terminated_length": 143.0,
|
|
"completions/mean_length": 22.40625,
|
|
"completions/mean_terminated_length": 22.40625,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.07366071428571429,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 25.130882263183594,
|
|
"learning_rate": 8.135881792367685e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3683175.0,
|
|
"reward": 0.33281251788139343,
|
|
"reward_std": 0.19129052758216858,
|
|
"rewards/format_reward/mean": 0.984375,
|
|
"rewards/format_reward/std": 0.125,
|
|
"rewards/mcq_exact_match_reward/mean": 0.234375,
|
|
"rewards/mcq_exact_match_reward/std": 0.42695629596710205,
|
|
"step": 33
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 20.0,
|
|
"completions/max_terminated_length": 20.0,
|
|
"completions/mean_length": 12.71875,
|
|
"completions/mean_terminated_length": 12.71875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.07589285714285714,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 10.324932098388672,
|
|
"learning_rate": 8.005405736415125e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3761261.0,
|
|
"reward": 0.8812500238418579,
|
|
"reward_std": 0.16675157845020294,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.78125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4166666865348816,
|
|
"step": 34
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 157.0,
|
|
"completions/max_terminated_length": 157.0,
|
|
"completions/mean_length": 14.390625,
|
|
"completions/mean_terminated_length": 14.390625,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"epoch": 0.078125,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 35.36943817138672,
|
|
"learning_rate": 7.871643313414718e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3860406.0,
|
|
"reward": 0.643750011920929,
|
|
"reward_std": 0.06803441047668457,
|
|
"rewards/format_reward/mean": 0.96875,
|
|
"rewards/format_reward/std": 0.12198751419782639,
|
|
"rewards/mcq_exact_match_reward/mean": 0.546875,
|
|
"rewards/mcq_exact_match_reward/std": 0.501733124256134,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 118.0,
|
|
"completions/max_terminated_length": 118.0,
|
|
"completions/mean_length": 16.1875,
|
|
"completions/mean_terminated_length": 16.1875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.08035714285714286,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 9.857135772705078,
|
|
"learning_rate": 7.734740790612136e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3950842.0,
|
|
"reward": 0.6625000238418579,
|
|
"reward_std": 0.1157275140285492,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.5625,
|
|
"rewards/mcq_exact_match_reward/std": 0.5,
|
|
"step": 36
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 565.0,
|
|
"completions/max_terminated_length": 565.0,
|
|
"completions/mean_length": 21.984375,
|
|
"completions/mean_terminated_length": 21.984375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.08258928571428571,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 17.679731369018555,
|
|
"learning_rate": 7.594847868906076e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4056305.0,
|
|
"reward": 0.5843750238418579,
|
|
"reward_std": 0.17358146607875824,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.484375,
|
|
"rewards/mcq_exact_match_reward/std": 0.5037065148353577,
|
|
"step": 37
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 664.0,
|
|
"completions/max_terminated_length": 664.0,
|
|
"completions/mean_length": 27.4375,
|
|
"completions/mean_terminated_length": 27.4375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.08482142857142858,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 7.502120494842529,
|
|
"learning_rate": 7.452117519152541e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4151749.0,
|
|
"reward": 0.3812500238418579,
|
|
"reward_std": 0.1462521106004715,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.28125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4531635046005249,
|
|
"step": 38
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 19.0,
|
|
"completions/max_terminated_length": 19.0,
|
|
"completions/mean_length": 12.109375,
|
|
"completions/mean_terminated_length": 12.109375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.08705357142857142,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 7.0521392822265625,
|
|
"learning_rate": 7.306705814893439e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4230140.0,
|
|
"reward": 0.5828125476837158,
|
|
"reward_std": 0.04861358925700188,
|
|
"rewards/format_reward/mean": 0.984375,
|
|
"rewards/format_reward/std": 0.125,
|
|
"rewards/mcq_exact_match_reward/mean": 0.484375,
|
|
"rewards/mcq_exact_match_reward/std": 0.5037065148353577,
|
|
"step": 39
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 875.0,
|
|
"completions/max_terminated_length": 875.0,
|
|
"completions/mean_length": 32.421875,
|
|
"completions/mean_terminated_length": 32.421875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.08928571428571429,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 11.019020080566406,
|
|
"learning_rate": 7.158771761692464e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4325367.0,
|
|
"reward": 0.6312500238418579,
|
|
"reward_std": 0.10888782143592834,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.53125,
|
|
"rewards/mcq_exact_match_reward/std": 0.5029674172401428,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 66.0,
|
|
"completions/max_terminated_length": 66.0,
|
|
"completions/mean_length": 13.0,
|
|
"completions/mean_terminated_length": 13.0,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.09151785714285714,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.008477123264847e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4422711.0,
|
|
"reward": 0.7250000238418579,
|
|
"reward_std": 0.0,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.625,
|
|
"rewards/mcq_exact_match_reward/std": 0.48795005679130554,
|
|
"step": 41
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 44.0,
|
|
"completions/max_terminated_length": 44.0,
|
|
"completions/mean_length": 13.265625,
|
|
"completions/mean_terminated_length": 13.265625,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.09375,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 17.952268600463867,
|
|
"learning_rate": 6.855986244591103e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4507048.0,
|
|
"reward": 0.5296875238418579,
|
|
"reward_std": 0.12682846188545227,
|
|
"rewards/format_reward/mean": 0.921875,
|
|
"rewards/format_reward/std": 0.27048972249031067,
|
|
"rewards/mcq_exact_match_reward/mean": 0.4375,
|
|
"rewards/mcq_exact_match_reward/std": 0.5,
|
|
"step": 42
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.0,
|
|
"completions/max_terminated_length": 296.0,
|
|
"completions/mean_length": 18.1875,
|
|
"completions/mean_terminated_length": 18.1875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.09598214285714286,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 5.754403591156006,
|
|
"learning_rate": 6.701465872208216e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4605428.0,
|
|
"reward": 0.7093750238418579,
|
|
"reward_std": 0.10205793380737305,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.609375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4917473793029785,
|
|
"step": 43
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1060.0,
|
|
"completions/max_terminated_length": 1060.0,
|
|
"completions/mean_length": 41.875,
|
|
"completions/mean_terminated_length": 41.875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.09821428571428571,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 18.143627166748047,
|
|
"learning_rate": 6.545084971874736e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4702924.0,
|
|
"reward": 0.5671875476837158,
|
|
"reward_std": 0.22903475165367126,
|
|
"rewards/format_reward/mean": 0.984375,
|
|
"rewards/format_reward/std": 0.125,
|
|
"rewards/mcq_exact_match_reward/mean": 0.46875,
|
|
"rewards/mcq_exact_match_reward/std": 0.5029674172401428,
|
|
"step": 44
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 19.0,
|
|
"completions/max_terminated_length": 19.0,
|
|
"completions/mean_length": 14.015625,
|
|
"completions/mean_terminated_length": 14.015625,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.10044642857142858,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 12.582730293273926,
|
|
"learning_rate": 6.387014543809223e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 4826389.0,
|
|
"reward": 0.6312500238418579,
|
|
"reward_std": 0.1462521106004715,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.53125,
|
|
"rewards/mcq_exact_match_reward/std": 0.5029674172401428,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 2048.0,
|
|
"completions/max_terminated_length": 115.0,
|
|
"completions/mean_length": 48.5625,
|
|
"completions/mean_terminated_length": 16.825397491455078,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.10267857142857142,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 29.780662536621094,
|
|
"learning_rate": 6.227427435703995e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 4927657.0,
|
|
"reward": 0.45781251788139343,
|
|
"reward_std": 0.26638925075531006,
|
|
"rewards/format_reward/mean": 0.984375,
|
|
"rewards/format_reward/std": 0.125,
|
|
"rewards/mcq_exact_match_reward/mean": 0.359375,
|
|
"rewards/mcq_exact_match_reward/std": 0.4836103618144989,
|
|
"step": 46
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 656.0,
|
|
"completions/max_terminated_length": 656.0,
|
|
"completions/mean_length": 29.828125,
|
|
"completions/mean_terminated_length": 29.828125,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.10491071428571429,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 8.41478443145752,
|
|
"learning_rate": 6.066498153718734e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 5018206.0,
|
|
"reward": 0.6937500238418579,
|
|
"reward_std": 0.1246790662407875,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.59375,
|
|
"rewards/mcq_exact_match_reward/std": 0.49501484632492065,
|
|
"step": 47
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 478.0,
|
|
"completions/max_terminated_length": 478.0,
|
|
"completions/mean_length": 20.40625,
|
|
"completions/mean_terminated_length": 20.40625,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.10714285714285714,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 10.784804344177246,
|
|
"learning_rate": 5.90440267166055e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 5120456.0,
|
|
"reward": 0.5375000238418579,
|
|
"reward_std": 0.10888782143592834,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.4375,
|
|
"rewards/mcq_exact_match_reward/std": 0.5,
|
|
"step": 48
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 18.0,
|
|
"completions/max_terminated_length": 18.0,
|
|
"completions/mean_length": 12.625,
|
|
"completions/mean_terminated_length": 12.625,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.109375,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 21.239360809326172,
|
|
"learning_rate": 5.741318238559209e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 5192800.0,
|
|
"reward": 0.870312511920929,
|
|
"reward_std": 0.24963238835334778,
|
|
"rewards/format_reward/mean": 0.890625,
|
|
"rewards/format_reward/std": 0.3145764470100403,
|
|
"rewards/mcq_exact_match_reward/mean": 0.78125,
|
|
"rewards/mcq_exact_match_reward/std": 0.4166666865348816,
|
|
"step": 49
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 113.0,
|
|
"completions/max_terminated_length": 113.0,
|
|
"completions/mean_length": 15.75,
|
|
"completions/mean_terminated_length": 15.75,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"epoch": 0.11160714285714286,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 18.710081100463867,
|
|
"learning_rate": 5.577423184847931e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 5301304.0,
|
|
"reward": 0.4750000238418579,
|
|
"reward_std": 0.2709311842918396,
|
|
"rewards/format_reward/mean": 1.0,
|
|
"rewards/format_reward/std": 0.0,
|
|
"rewards/mcq_exact_match_reward/mean": 0.375,
|
|
"rewards/mcq_exact_match_reward/std": 0.48795005679130554,
|
|
"step": 50
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 100,
|
|
"num_input_tokens_seen": 5301304,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|