40535 lines
1.3 MiB
40535 lines
1.3 MiB
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8698173383589446,
|
|
"eval_steps": 500,
|
|
"global_step": 1500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1425.0,
|
|
"completions/mean_length": 638.59375,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.0005798782255726297,
|
|
"grad_norm": 0.47989900463476126,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.807692307692308e-09,
|
|
"loss": 5.587935447692871e-09,
|
|
"memory(GiB)": 41.18,
|
|
"reward": 1.09375,
|
|
"reward_std": 0.6235843896865845,
|
|
"rewards/CSTORM/mean": 0.17578125,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.42578125,
|
|
"rewards/FMTORM/std": 0.1784650683403015,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 1,
|
|
"train_speed(iter/s)": 0.004014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2177.0,
|
|
"completions/mean_length": 674.5625,
|
|
"completions/min_length": 194.0,
|
|
"epoch": 0.0011597564511452595,
|
|
"grad_norm": 0.4419081314960068,
|
|
"kl": 0.0,
|
|
"learning_rate": 9.615384615384615e-09,
|
|
"loss": 0.0,
|
|
"memory(GiB)": 43.21,
|
|
"reward": 0.96875,
|
|
"reward_std": 0.691143274307251,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.39453125,
|
|
"rewards/FMTORM/std": 0.20478858053684235,
|
|
"rewards/VQAORM/mean": 0.421875,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 2,
|
|
"train_speed(iter/s)": 0.006227
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1440.0,
|
|
"completions/mean_length": 592.7265625,
|
|
"completions/min_length": 156.0,
|
|
"epoch": 0.0017396346767178893,
|
|
"grad_norm": 0.4327720388292178,
|
|
"kl": 1.52587890625e-05,
|
|
"learning_rate": 1.4423076923076924e-08,
|
|
"loss": 9.987640936515163e-09,
|
|
"memory(GiB)": 49.81,
|
|
"reward": 1.29296875,
|
|
"reward_std": 0.7417604923248291,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 3,
|
|
"train_speed(iter/s)": 0.008072
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2066.0,
|
|
"completions/mean_length": 687.734375,
|
|
"completions/min_length": 210.0,
|
|
"epoch": 0.002319512902290519,
|
|
"grad_norm": 0.407886062063289,
|
|
"kl": 2.378225326538086e-05,
|
|
"learning_rate": 1.923076923076923e-08,
|
|
"loss": 1.6400299784891104e-08,
|
|
"memory(GiB)": 49.81,
|
|
"reward": 0.88671875,
|
|
"reward_std": 0.5635688304901123,
|
|
"rewards/CSTORM/mean": 0.125,
|
|
"rewards/CSTORM/std": 0.2173570692539215,
|
|
"rewards/FMTORM/mean": 0.38671875,
|
|
"rewards/FMTORM/std": 0.21012598276138306,
|
|
"rewards/VQAORM/mean": 0.375,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 4,
|
|
"train_speed(iter/s)": 0.008819
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1207.0,
|
|
"completions/mean_length": 594.7265625,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.0028993911278631487,
|
|
"grad_norm": 0.395354957184368,
|
|
"kl": 2.181529998779297e-05,
|
|
"learning_rate": 2.403846153846154e-08,
|
|
"loss": 1.6652791146043455e-08,
|
|
"memory(GiB)": 49.81,
|
|
"reward": 1.0703125,
|
|
"reward_std": 0.5942456722259521,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.39453125,
|
|
"rewards/FMTORM/std": 0.20478858053684235,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 5,
|
|
"train_speed(iter/s)": 0.008629
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2133.0,
|
|
"completions/mean_length": 628.1796875,
|
|
"completions/min_length": 205.0,
|
|
"epoch": 0.0034792693534357786,
|
|
"grad_norm": 0.4681752864963699,
|
|
"kl": 1.7881393432617188e-05,
|
|
"learning_rate": 2.8846153846153848e-08,
|
|
"loss": 1.5832483768463135e-08,
|
|
"memory(GiB)": 49.81,
|
|
"reward": 1.0,
|
|
"reward_std": 0.5442265272140503,
|
|
"rewards/CSTORM/mean": 0.1484375,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.3984375,
|
|
"rewards/FMTORM/std": 0.20195281505584717,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 6,
|
|
"train_speed(iter/s)": 0.009247
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2336.0,
|
|
"completions/mean_length": 630.078125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.004059147579008408,
|
|
"grad_norm": 0.494575809796329,
|
|
"kl": 3.2901763916015625e-05,
|
|
"learning_rate": 3.365384615384615e-08,
|
|
"loss": 3.3674645294468064e-08,
|
|
"memory(GiB)": 49.81,
|
|
"reward": 1.07421875,
|
|
"reward_std": 0.606322169303894,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.4140625,
|
|
"rewards/FMTORM/std": 0.1893770843744278,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 7,
|
|
"train_speed(iter/s)": 0.009689
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1545.0,
|
|
"completions/mean_length": 619.6875,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.004639025804581038,
|
|
"grad_norm": 0.48643303438355806,
|
|
"kl": 3.8504600524902344e-05,
|
|
"learning_rate": 3.846153846153846e-08,
|
|
"loss": 4.190951585769653e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.01171875,
|
|
"reward_std": 0.5167067646980286,
|
|
"rewards/CSTORM/mean": 0.12890625,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 8,
|
|
"train_speed(iter/s)": 0.007294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 4612.0,
|
|
"completions/mean_length": 646.0078125,
|
|
"completions/min_length": 161.0,
|
|
"epoch": 0.005218904030153668,
|
|
"grad_norm": 0.4433972261411303,
|
|
"kl": 2.8371810913085938e-05,
|
|
"learning_rate": 4.326923076923076e-08,
|
|
"loss": 2.8837675358772685e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.92578125,
|
|
"reward_std": 0.516644299030304,
|
|
"rewards/CSTORM/mean": 0.11328125,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.40625,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 9,
|
|
"train_speed(iter/s)": 0.006917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2562.0,
|
|
"completions/mean_length": 671.6171875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.0057987822557262975,
|
|
"grad_norm": 0.49258928634934857,
|
|
"kl": 3.069639205932617e-05,
|
|
"learning_rate": 4.807692307692308e-08,
|
|
"loss": 2.8405338525772095e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.921875,
|
|
"reward_std": 0.5513356924057007,
|
|
"rewards/CSTORM/mean": 0.1484375,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.375,
|
|
"rewards/FMTORM/std": 0.2173570692539215,
|
|
"rewards/VQAORM/mean": 0.3984375,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 10,
|
|
"train_speed(iter/s)": 0.007219
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1529.0,
|
|
"completions/mean_length": 660.453125,
|
|
"completions/min_length": 141.0,
|
|
"epoch": 0.006378660481298927,
|
|
"grad_norm": 0.5202306237522256,
|
|
"kl": 3.3736228942871094e-05,
|
|
"learning_rate": 5.288461538461538e-08,
|
|
"loss": 3.725290298461914e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.9609375,
|
|
"reward_std": 0.648123025894165,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.37109375,
|
|
"rewards/FMTORM/std": 0.21957451105117798,
|
|
"rewards/VQAORM/mean": 0.4375,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 11,
|
|
"train_speed(iter/s)": 0.007577
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1679.0,
|
|
"completions/mean_length": 637.8671875,
|
|
"completions/min_length": 142.0,
|
|
"epoch": 0.006958538706871557,
|
|
"grad_norm": 0.44902401963496735,
|
|
"kl": 2.6285648345947266e-05,
|
|
"learning_rate": 5.7692307692307695e-08,
|
|
"loss": 2.3506352420099574e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.2265625,
|
|
"reward_std": 0.5663236379623413,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.41796875,
|
|
"rewards/FMTORM/std": 0.185893714427948,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 12,
|
|
"train_speed(iter/s)": 0.007206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 7806.0,
|
|
"completions/mean_length": 742.1328125,
|
|
"completions/min_length": 227.0,
|
|
"epoch": 0.007538416932444187,
|
|
"grad_norm": 0.4527324460978441,
|
|
"kl": 2.1219253540039062e-05,
|
|
"learning_rate": 6.25e-08,
|
|
"loss": 2.561137080192566e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.87109375,
|
|
"reward_std": 0.6457931995391846,
|
|
"rewards/CSTORM/mean": 0.13671875,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.3671875,
|
|
"rewards/FMTORM/std": 0.22170042991638184,
|
|
"rewards/VQAORM/mean": 0.3671875,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 13,
|
|
"train_speed(iter/s)": 0.006863
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2489.0,
|
|
"completions/mean_length": 647.6875,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.008118295158016816,
|
|
"grad_norm": 0.47149072807316766,
|
|
"kl": 2.676248550415039e-05,
|
|
"learning_rate": 6.73076923076923e-08,
|
|
"loss": 2.3283064365386963e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.984375,
|
|
"reward_std": 0.6124204397201538,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.39453125,
|
|
"rewards/FMTORM/std": 0.20478858053684235,
|
|
"rewards/VQAORM/mean": 0.4375,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 14,
|
|
"train_speed(iter/s)": 0.00707
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 4452.0,
|
|
"completions/mean_length": 668.984375,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.008698173383589447,
|
|
"grad_norm": 0.4386064434332235,
|
|
"kl": 3.129243850708008e-05,
|
|
"learning_rate": 7.21153846153846e-08,
|
|
"loss": 3.371072665458996e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.90625,
|
|
"reward_std": 0.5702477097511292,
|
|
"rewards/CSTORM/mean": 0.140625,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.390625,
|
|
"rewards/FMTORM/std": 0.20751149952411652,
|
|
"rewards/VQAORM/mean": 0.375,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 15,
|
|
"train_speed(iter/s)": 0.007085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1428.0,
|
|
"completions/mean_length": 569.8984375,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.009278051609162076,
|
|
"grad_norm": 0.45956879713778387,
|
|
"kl": 4.0650367736816406e-05,
|
|
"learning_rate": 7.692307692307692e-08,
|
|
"loss": 4.668254405260086e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0625,
|
|
"reward_std": 0.5685778856277466,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.4296875,
|
|
"rewards/FMTORM/std": 0.17450013756752014,
|
|
"rewards/VQAORM/mean": 0.4609375,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 16,
|
|
"train_speed(iter/s)": 0.007338
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1379.0,
|
|
"completions/mean_length": 629.8828125,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.009857929834734706,
|
|
"grad_norm": 0.4587131072692109,
|
|
"kl": 2.2709369659423828e-05,
|
|
"learning_rate": 8.173076923076923e-08,
|
|
"loss": 2.963442469194888e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.06640625,
|
|
"reward_std": 0.6376256942749023,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.38671875,
|
|
"rewards/FMTORM/std": 0.21012598276138306,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 17,
|
|
"train_speed(iter/s)": 0.007592
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1515.0,
|
|
"completions/mean_length": 606.09375,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.010437808060307335,
|
|
"grad_norm": 0.4166305867992526,
|
|
"kl": 3.695487976074219e-05,
|
|
"learning_rate": 8.653846153846153e-08,
|
|
"loss": 3.725290298461914e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.9375,
|
|
"reward_std": 0.4812837243080139,
|
|
"rewards/CSTORM/mean": 0.1328125,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.3984375,
|
|
"rewards/FMTORM/std": 0.20195281505584717,
|
|
"rewards/VQAORM/mean": 0.40625,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 18,
|
|
"train_speed(iter/s)": 0.007612
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2960.0,
|
|
"completions/mean_length": 637.96875,
|
|
"completions/min_length": 193.0,
|
|
"epoch": 0.011017686285879966,
|
|
"grad_norm": 0.46991571364501034,
|
|
"kl": 2.7179718017578125e-05,
|
|
"learning_rate": 9.134615384615383e-08,
|
|
"loss": 3.1490188234784e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.921875,
|
|
"reward_std": 0.5243874788284302,
|
|
"rewards/CSTORM/mean": 0.12890625,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.38671875,
|
|
"rewards/FMTORM/std": 0.21012598276138306,
|
|
"rewards/VQAORM/mean": 0.40625,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 19,
|
|
"train_speed(iter/s)": 0.007727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1284.0,
|
|
"completions/mean_length": 624.7421875,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.011597564511452595,
|
|
"grad_norm": 0.42455772820447363,
|
|
"kl": 3.3736228942871094e-05,
|
|
"learning_rate": 9.615384615384616e-08,
|
|
"loss": 3.241620305516335e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0078125,
|
|
"reward_std": 0.5298221707344055,
|
|
"rewards/CSTORM/mean": 0.12109375,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.37890625,
|
|
"rewards/FMTORM/std": 0.2150452584028244,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 20,
|
|
"train_speed(iter/s)": 0.007957
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1623.0,
|
|
"completions/mean_length": 604.46875,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.012177442737025224,
|
|
"grad_norm": 0.45283915308185874,
|
|
"kl": 2.4616718292236328e-05,
|
|
"learning_rate": 1.0096153846153847e-07,
|
|
"loss": 2.6542693376541138e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.140625,
|
|
"reward_std": 0.5166463851928711,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.4375,
|
|
"rewards/FMTORM/std": 0.16600920259952545,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 21,
|
|
"train_speed(iter/s)": 0.008139
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2988.0,
|
|
"completions/mean_length": 637.2421875,
|
|
"completions/min_length": 182.0,
|
|
"epoch": 0.012757320962597855,
|
|
"grad_norm": 0.44345513232906286,
|
|
"kl": 2.2590160369873047e-05,
|
|
"learning_rate": 1.0576923076923076e-07,
|
|
"loss": 2.7935655566579953e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.16015625,
|
|
"reward_std": 0.6169758439064026,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.4140625,
|
|
"rewards/FMTORM/std": 0.1893770843744278,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 22,
|
|
"train_speed(iter/s)": 0.007389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1869.0,
|
|
"completions/mean_length": 621.484375,
|
|
"completions/min_length": 63.0,
|
|
"epoch": 0.013337199188170484,
|
|
"grad_norm": 0.49448683303161023,
|
|
"kl": 4.1365623474121094e-05,
|
|
"learning_rate": 1.1057692307692307e-07,
|
|
"loss": 4.579166557050485e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0234375,
|
|
"reward_std": 0.5837802290916443,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.3828125,
|
|
"rewards/FMTORM/std": 0.21263602375984192,
|
|
"rewards/VQAORM/mean": 0.46875,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 23,
|
|
"train_speed(iter/s)": 0.006862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2079.0,
|
|
"completions/mean_length": 634.3359375,
|
|
"completions/min_length": 158.0,
|
|
"epoch": 0.013917077413743114,
|
|
"grad_norm": 0.44142938491181144,
|
|
"kl": 3.1948089599609375e-05,
|
|
"learning_rate": 1.1538461538461539e-07,
|
|
"loss": 3.329478204250336e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.93359375,
|
|
"reward_std": 0.5570834875106812,
|
|
"rewards/CSTORM/mean": 0.140625,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.36328125,
|
|
"rewards/FMTORM/std": 0.223737433552742,
|
|
"rewards/VQAORM/mean": 0.4296875,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 24,
|
|
"train_speed(iter/s)": 0.006424
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1473.0,
|
|
"completions/mean_length": 600.2421875,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.014496955639315743,
|
|
"grad_norm": 0.40892959923222055,
|
|
"kl": 2.950429916381836e-05,
|
|
"learning_rate": 1.2019230769230769e-07,
|
|
"loss": 2.8722368483613536e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0234375,
|
|
"reward_std": 0.42132115364074707,
|
|
"rewards/CSTORM/mean": 0.140625,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.4296875,
|
|
"rewards/FMTORM/std": 0.17450013756752014,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 25,
|
|
"train_speed(iter/s)": 0.006559
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 627.7578125,
|
|
"completions/min_length": 174.0,
|
|
"epoch": 0.015076833864888374,
|
|
"grad_norm": 0.5069195311903639,
|
|
"kl": 3.314018249511719e-05,
|
|
"learning_rate": 1.25e-07,
|
|
"loss": 3.5157427191734314e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.109375,
|
|
"reward_std": 0.5008096694946289,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.42578125,
|
|
"rewards/FMTORM/std": 0.1784650683403015,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 26,
|
|
"train_speed(iter/s)": 0.006088
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2428.0,
|
|
"completions/mean_length": 660.4453125,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.015656712090461003,
|
|
"grad_norm": 0.49839395689834654,
|
|
"kl": 2.9325485229492188e-05,
|
|
"learning_rate": 1.298076923076923e-07,
|
|
"loss": 2.8405338525772095e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.07421875,
|
|
"reward_std": 0.5871164798736572,
|
|
"rewards/CSTORM/mean": 0.16796875,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 27,
|
|
"train_speed(iter/s)": 0.006209
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1496.0,
|
|
"completions/mean_length": 576.984375,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.016236590316033632,
|
|
"grad_norm": 0.5241374034023686,
|
|
"kl": 3.2961368560791016e-05,
|
|
"learning_rate": 1.346153846153846e-07,
|
|
"loss": 3.005259685551209e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0,
|
|
"reward_std": 0.581149160861969,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.41796875,
|
|
"rewards/FMTORM/std": 0.185893714427948,
|
|
"rewards/VQAORM/mean": 0.4296875,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 28,
|
|
"train_speed(iter/s)": 0.006357
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1656.0,
|
|
"completions/mean_length": 632.7578125,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.016816468541606264,
|
|
"grad_norm": 0.39230256078921827,
|
|
"kl": 2.8014183044433594e-05,
|
|
"learning_rate": 1.3942307692307692e-07,
|
|
"loss": 2.4616745974981313e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.12109375,
|
|
"reward_std": 0.6302359104156494,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.390625,
|
|
"rewards/FMTORM/std": 0.20751149952411652,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 29,
|
|
"train_speed(iter/s)": 0.006494
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1276.0,
|
|
"completions/mean_length": 578.15625,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.017396346767178893,
|
|
"grad_norm": 0.4349250429443359,
|
|
"kl": 3.886222839355469e-05,
|
|
"learning_rate": 1.442307692307692e-07,
|
|
"loss": 4.0745362639427185e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.11328125,
|
|
"reward_std": 0.5516945123672485,
|
|
"rewards/CSTORM/mean": 0.1484375,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.43359375,
|
|
"rewards/FMTORM/std": 0.1703527420759201,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 30,
|
|
"train_speed(iter/s)": 0.006643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1494.0,
|
|
"completions/mean_length": 610.1015625,
|
|
"completions/min_length": 228.0,
|
|
"epoch": 0.017976224992751522,
|
|
"grad_norm": 0.4243106394550288,
|
|
"kl": 3.0040740966796875e-05,
|
|
"learning_rate": 1.4903846153846154e-07,
|
|
"loss": 3.339411591696262e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.953125,
|
|
"reward_std": 0.5511713027954102,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.37890625,
|
|
"rewards/FMTORM/std": 0.2150452584028244,
|
|
"rewards/VQAORM/mean": 0.421875,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 31,
|
|
"train_speed(iter/s)": 0.006322
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1352.0,
|
|
"completions/mean_length": 639.9140625,
|
|
"completions/min_length": 146.0,
|
|
"epoch": 0.01855610321832415,
|
|
"grad_norm": 0.42762436399292175,
|
|
"kl": 3.325939178466797e-05,
|
|
"learning_rate": 1.5384615384615385e-07,
|
|
"loss": 2.991873770952225e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.99609375,
|
|
"reward_std": 0.6697455644607544,
|
|
"rewards/CSTORM/mean": 0.13671875,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 32,
|
|
"train_speed(iter/s)": 0.006436
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1219.0,
|
|
"completions/mean_length": 624.3984375,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.01913598144389678,
|
|
"grad_norm": 0.45321789487836694,
|
|
"kl": 3.9458274841308594e-05,
|
|
"learning_rate": 1.5865384615384613e-07,
|
|
"loss": 3.4093147860403406e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.99609375,
|
|
"reward_std": 0.5994052290916443,
|
|
"rewards/CSTORM/mean": 0.1640625,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.38671875,
|
|
"rewards/FMTORM/std": 0.21012598276138306,
|
|
"rewards/VQAORM/mean": 0.4453125,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 33,
|
|
"train_speed(iter/s)": 0.006371
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1678.0,
|
|
"completions/mean_length": 602.2890625,
|
|
"completions/min_length": 61.0,
|
|
"epoch": 0.019715859669469413,
|
|
"grad_norm": 0.4760505994161235,
|
|
"kl": 4.9591064453125e-05,
|
|
"learning_rate": 1.6346153846153846e-07,
|
|
"loss": 4.788648055864542e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.03515625,
|
|
"reward_std": 0.5619536638259888,
|
|
"rewards/CSTORM/mean": 0.16796875,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.3984375,
|
|
"rewards/FMTORM/std": 0.20195281505584717,
|
|
"rewards/VQAORM/mean": 0.46875,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 34,
|
|
"train_speed(iter/s)": 0.006488
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1464.0,
|
|
"completions/mean_length": 631.921875,
|
|
"completions/min_length": 194.0,
|
|
"epoch": 0.02029573789504204,
|
|
"grad_norm": 0.41203350683728945,
|
|
"kl": 4.398822784423828e-05,
|
|
"learning_rate": 1.6826923076923077e-07,
|
|
"loss": 4.886877036369697e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.015625,
|
|
"reward_std": 0.5637578964233398,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.38671875,
|
|
"rewards/FMTORM/std": 0.21012598276138306,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 35,
|
|
"train_speed(iter/s)": 0.006229
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1556.0,
|
|
"completions/mean_length": 634.265625,
|
|
"completions/min_length": 169.0,
|
|
"epoch": 0.02087561612061467,
|
|
"grad_norm": 0.4538100261142054,
|
|
"kl": 5.447864532470703e-05,
|
|
"learning_rate": 1.7307692307692305e-07,
|
|
"loss": 5.601668817689642e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.98828125,
|
|
"reward_std": 0.5067128539085388,
|
|
"rewards/CSTORM/mean": 0.13671875,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.375,
|
|
"rewards/FMTORM/std": 0.2173570692539215,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 36,
|
|
"train_speed(iter/s)": 0.006003
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1601.0,
|
|
"completions/mean_length": 588.21875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.0214554943461873,
|
|
"grad_norm": 0.5260065664631298,
|
|
"kl": 6.103515625e-05,
|
|
"learning_rate": 1.778846153846154e-07,
|
|
"loss": 6.07342371949926e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0859375,
|
|
"reward_std": 0.6154720783233643,
|
|
"rewards/CSTORM/mean": 0.14453125,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.41796875,
|
|
"rewards/FMTORM/std": 0.185893714427948,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 37,
|
|
"train_speed(iter/s)": 0.006106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1416.0,
|
|
"completions/mean_length": 607.515625,
|
|
"completions/min_length": 199.0,
|
|
"epoch": 0.022035372571759932,
|
|
"grad_norm": 0.43458243171242006,
|
|
"kl": 4.6133995056152344e-05,
|
|
"learning_rate": 1.8269230769230767e-07,
|
|
"loss": 4.4879310934220484e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.078125,
|
|
"reward_std": 0.6163773536682129,
|
|
"rewards/CSTORM/mean": 0.1796875,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 38,
|
|
"train_speed(iter/s)": 0.005965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1493.0,
|
|
"completions/mean_length": 646.7421875,
|
|
"completions/min_length": 139.0,
|
|
"epoch": 0.02261525079733256,
|
|
"grad_norm": 0.41586684475005886,
|
|
"kl": 6.628036499023438e-05,
|
|
"learning_rate": 1.875e-07,
|
|
"loss": 7.140552327200567e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.94921875,
|
|
"reward_std": 0.497367799282074,
|
|
"rewards/CSTORM/mean": 0.140625,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.36328125,
|
|
"rewards/FMTORM/std": 0.223737433552742,
|
|
"rewards/VQAORM/mean": 0.4453125,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 39,
|
|
"train_speed(iter/s)": 0.006072
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1526.0,
|
|
"completions/mean_length": 593.4453125,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.02319512902290519,
|
|
"grad_norm": 0.48153925036707523,
|
|
"kl": 6.186962127685547e-05,
|
|
"learning_rate": 1.9230769230769231e-07,
|
|
"loss": 6.100162863731384e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.984375,
|
|
"reward_std": 0.5385758876800537,
|
|
"rewards/CSTORM/mean": 0.1328125,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.421875,
|
|
"rewards/FMTORM/std": 0.1822594404220581,
|
|
"rewards/VQAORM/mean": 0.4296875,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 40,
|
|
"train_speed(iter/s)": 0.006164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/mean_length": 624.6484375,
|
|
"completions/min_length": 194.0,
|
|
"epoch": 0.02377500724847782,
|
|
"grad_norm": 0.38464399555770185,
|
|
"kl": 4.982948303222656e-05,
|
|
"learning_rate": 1.971153846153846e-07,
|
|
"loss": 5.0555158281895274e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.078125,
|
|
"reward_std": 0.49261635541915894,
|
|
"rewards/CSTORM/mean": 0.17578125,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.40234375,
|
|
"rewards/FMTORM/std": 0.19899940490722656,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 41,
|
|
"train_speed(iter/s)": 0.006265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2052.0,
|
|
"completions/mean_length": 589.3515625,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.024354885474050448,
|
|
"grad_norm": 7.140450126590614,
|
|
"kl": 0.00013399124145507812,
|
|
"learning_rate": 2.0192307692307693e-07,
|
|
"loss": 1.3789413344511559e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.06640625,
|
|
"reward_std": 0.5893666744232178,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.42578125,
|
|
"rewards/FMTORM/std": 0.1784650683403015,
|
|
"rewards/VQAORM/mean": 0.46875,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 42,
|
|
"train_speed(iter/s)": 0.006001
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1187.0,
|
|
"completions/mean_length": 634.5,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.02493476369962308,
|
|
"grad_norm": 0.46222240194990627,
|
|
"kl": 9.655952453613281e-05,
|
|
"learning_rate": 2.067307692307692e-07,
|
|
"loss": 9.884116991543124e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0859375,
|
|
"reward_std": 0.5176330208778381,
|
|
"rewards/CSTORM/mean": 0.17578125,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.39453125,
|
|
"rewards/FMTORM/std": 0.20478858053684235,
|
|
"rewards/VQAORM/mean": 0.515625,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 43,
|
|
"train_speed(iter/s)": 0.006047
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1349.0,
|
|
"completions/mean_length": 587.2734375,
|
|
"completions/min_length": 215.0,
|
|
"epoch": 0.02551464192519571,
|
|
"grad_norm": 0.4004777834260277,
|
|
"kl": 7.867813110351562e-05,
|
|
"learning_rate": 2.1153846153846152e-07,
|
|
"loss": 8.355010550076258e-08,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1484375,
|
|
"reward_std": 0.4704464077949524,
|
|
"rewards/CSTORM/mean": 0.17578125,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.44140625,
|
|
"rewards/FMTORM/std": 0.1614537090063095,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 44,
|
|
"train_speed(iter/s)": 0.005944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1442.0,
|
|
"completions/mean_length": 652.9765625,
|
|
"completions/min_length": 216.0,
|
|
"epoch": 0.02609452015076834,
|
|
"grad_norm": 0.42877492758749086,
|
|
"kl": 0.00016069412231445312,
|
|
"learning_rate": 2.1634615384615386e-07,
|
|
"loss": 1.6387073742407665e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.99609375,
|
|
"reward_std": 0.6108126044273376,
|
|
"rewards/CSTORM/mean": 0.140625,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.37109375,
|
|
"rewards/FMTORM/std": 0.21957451105117798,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 45,
|
|
"train_speed(iter/s)": 0.00603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2638.0,
|
|
"completions/mean_length": 624.59375,
|
|
"completions/min_length": 59.0,
|
|
"epoch": 0.026674398376340967,
|
|
"grad_norm": 0.4999372242730832,
|
|
"kl": 0.00018978118896484375,
|
|
"learning_rate": 2.2115384615384614e-07,
|
|
"loss": 1.8894986908435385e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.93359375,
|
|
"reward_std": 0.4864196181297302,
|
|
"rewards/CSTORM/mean": 0.12109375,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.390625,
|
|
"rewards/FMTORM/std": 0.20751149952411652,
|
|
"rewards/VQAORM/mean": 0.421875,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 46,
|
|
"train_speed(iter/s)": 0.006092
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 5802.0,
|
|
"completions/mean_length": 642.953125,
|
|
"completions/min_length": 178.0,
|
|
"epoch": 0.0272542766019136,
|
|
"grad_norm": 0.42377678153904486,
|
|
"kl": 0.0002079010009765625,
|
|
"learning_rate": 2.2596153846153845e-07,
|
|
"loss": 2.1665783833668684e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.07421875,
|
|
"reward_std": 0.5672264099121094,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.3828125,
|
|
"rewards/FMTORM/std": 0.21263602375984192,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 47,
|
|
"train_speed(iter/s)": 0.006081
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1044.0,
|
|
"completions/mean_length": 601.078125,
|
|
"completions/min_length": 154.0,
|
|
"epoch": 0.02783415482748623,
|
|
"grad_norm": 0.48090093977960047,
|
|
"kl": 0.00016164779663085938,
|
|
"learning_rate": 2.3076923076923078e-07,
|
|
"loss": 1.6212479181376693e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0234375,
|
|
"reward_std": 0.5898147225379944,
|
|
"rewards/CSTORM/mean": 0.15625,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.3984375,
|
|
"rewards/FMTORM/std": 0.20195281505584717,
|
|
"rewards/VQAORM/mean": 0.46875,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 48,
|
|
"train_speed(iter/s)": 0.006109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1555.0,
|
|
"completions/mean_length": 691.6953125,
|
|
"completions/min_length": 134.0,
|
|
"epoch": 0.028414033053058858,
|
|
"grad_norm": 0.41994171694011395,
|
|
"kl": 0.00031375885009765625,
|
|
"learning_rate": 2.3557692307692306e-07,
|
|
"loss": 3.1875856620899867e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0703125,
|
|
"reward_std": 0.5734957456588745,
|
|
"rewards/CSTORM/mean": 0.140625,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.375,
|
|
"rewards/FMTORM/std": 0.2173570692539215,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 49,
|
|
"train_speed(iter/s)": 0.00619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1979.0,
|
|
"completions/mean_length": 635.7734375,
|
|
"completions/min_length": 169.0,
|
|
"epoch": 0.028993911278631487,
|
|
"grad_norm": 0.41168235092914335,
|
|
"kl": 0.0003833770751953125,
|
|
"learning_rate": 2.4038461538461537e-07,
|
|
"loss": 3.8289908843580633e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0390625,
|
|
"reward_std": 0.6477177739143372,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.37109375,
|
|
"rewards/FMTORM/std": 0.21957451105117798,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 50,
|
|
"train_speed(iter/s)": 0.006259
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1892.0,
|
|
"completions/mean_length": 651.5,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.029573789504204116,
|
|
"grad_norm": 0.4319835246483937,
|
|
"kl": 0.00021505355834960938,
|
|
"learning_rate": 2.4519230769230765e-07,
|
|
"loss": 2.164303651852606e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.11328125,
|
|
"reward_std": 0.47931763529777527,
|
|
"rewards/CSTORM/mean": 0.12109375,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.4296875,
|
|
"rewards/FMTORM/std": 0.17450013756752014,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 51,
|
|
"train_speed(iter/s)": 0.006332
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1920.0,
|
|
"completions/mean_length": 640.828125,
|
|
"completions/min_length": 235.0,
|
|
"epoch": 0.030153667729776748,
|
|
"grad_norm": 0.40520407022666177,
|
|
"kl": 0.000370025634765625,
|
|
"learning_rate": 2.5e-07,
|
|
"loss": 3.7034669730928726e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0546875,
|
|
"reward_std": 0.45913931727409363,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.3984375,
|
|
"rewards/FMTORM/std": 0.20195281505584717,
|
|
"rewards/VQAORM/mean": 0.46875,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 52,
|
|
"train_speed(iter/s)": 0.006401
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1397.0,
|
|
"completions/mean_length": 575.4765625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.030733545955349377,
|
|
"grad_norm": 0.47170204851688907,
|
|
"kl": 0.000385284423828125,
|
|
"learning_rate": 2.5480769230769227e-07,
|
|
"loss": 3.8385286416087183e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.2578125,
|
|
"reward_std": 0.6108803153038025,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.40234375,
|
|
"rewards/FMTORM/std": 0.19899940490722656,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 53,
|
|
"train_speed(iter/s)": 0.006236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1239.0,
|
|
"completions/mean_length": 580.875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.031313424180922006,
|
|
"grad_norm": 0.4918154687276096,
|
|
"kl": 0.0005426406860351562,
|
|
"learning_rate": 2.596153846153846e-07,
|
|
"loss": 5.360925570130348e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.09375,
|
|
"reward_std": 0.6105246543884277,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.390625,
|
|
"rewards/FMTORM/std": 0.20751149952411652,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 54,
|
|
"train_speed(iter/s)": 0.006307
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2832.0,
|
|
"completions/mean_length": 663.3203125,
|
|
"completions/min_length": 142.0,
|
|
"epoch": 0.031893302406494635,
|
|
"grad_norm": 0.4461779007227422,
|
|
"kl": 0.0008525848388671875,
|
|
"learning_rate": 2.6442307692307694e-07,
|
|
"loss": 8.598735234954802e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0078125,
|
|
"reward_std": 0.6264163851737976,
|
|
"rewards/CSTORM/mean": 0.17578125,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.39453125,
|
|
"rewards/FMTORM/std": 0.20478858053684235,
|
|
"rewards/VQAORM/mean": 0.4375,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 55,
|
|
"train_speed(iter/s)": 0.006261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2011.0,
|
|
"completions/mean_length": 688.3359375,
|
|
"completions/min_length": 190.0,
|
|
"epoch": 0.032473180632067264,
|
|
"grad_norm": 0.39504665131979505,
|
|
"kl": 0.00077056884765625,
|
|
"learning_rate": 2.692307692307692e-07,
|
|
"loss": 7.759654181427322e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.046875,
|
|
"reward_std": 0.5350997447967529,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.3984375,
|
|
"rewards/FMTORM/std": 0.20195281505584717,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 56,
|
|
"train_speed(iter/s)": 0.006103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1868.0,
|
|
"completions/mean_length": 587.1484375,
|
|
"completions/min_length": 155.0,
|
|
"epoch": 0.03305305885763989,
|
|
"grad_norm": 0.47814462597698043,
|
|
"kl": 0.0005626678466796875,
|
|
"learning_rate": 2.7403846153846156e-07,
|
|
"loss": 5.587031068898796e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.171875,
|
|
"reward_std": 0.6103599667549133,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.4296875,
|
|
"rewards/FMTORM/std": 0.17450013756752014,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 57,
|
|
"train_speed(iter/s)": 0.005968
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1455.0,
|
|
"completions/mean_length": 583.5078125,
|
|
"completions/min_length": 158.0,
|
|
"epoch": 0.03363293708321253,
|
|
"grad_norm": 0.4585464505996166,
|
|
"kl": 0.000667572021484375,
|
|
"learning_rate": 2.7884615384615384e-07,
|
|
"loss": 6.654299795627594e-07,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.9765625,
|
|
"reward_std": 0.562968373298645,
|
|
"rewards/CSTORM/mean": 0.140625,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.421875,
|
|
"rewards/FMTORM/std": 0.1822594404220581,
|
|
"rewards/VQAORM/mean": 0.4140625,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 58,
|
|
"train_speed(iter/s)": 0.005987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2394.0,
|
|
"completions/mean_length": 693.0078125,
|
|
"completions/min_length": 140.0,
|
|
"epoch": 0.03421281530878516,
|
|
"grad_norm": 0.44017317153125296,
|
|
"kl": 0.001293182373046875,
|
|
"learning_rate": 2.836538461538461e-07,
|
|
"loss": 1.2961758102392196e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.90625,
|
|
"reward_std": 0.6068457365036011,
|
|
"rewards/CSTORM/mean": 0.10546875,
|
|
"rewards/CSTORM/std": 0.20478858053684235,
|
|
"rewards/FMTORM/mean": 0.38671875,
|
|
"rewards/FMTORM/std": 0.21012598276138306,
|
|
"rewards/VQAORM/mean": 0.4140625,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 59,
|
|
"train_speed(iter/s)": 0.005964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1626.0,
|
|
"completions/mean_length": 614.71875,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.03479269353435779,
|
|
"grad_norm": 0.46029116933400055,
|
|
"kl": 0.001583099365234375,
|
|
"learning_rate": 2.884615384615384e-07,
|
|
"loss": 1.5824872434677673e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.05859375,
|
|
"reward_std": 0.5394303798675537,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.38671875,
|
|
"rewards/FMTORM/std": 0.21012598276138306,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 60,
|
|
"train_speed(iter/s)": 0.006028
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1142.0,
|
|
"completions/mean_length": 595.90625,
|
|
"completions/min_length": 152.0,
|
|
"epoch": 0.035372571759930416,
|
|
"grad_norm": 0.3958336904175345,
|
|
"kl": 0.0015411376953125,
|
|
"learning_rate": 2.932692307692308e-07,
|
|
"loss": 1.5426801383000566e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.12890625,
|
|
"reward_std": 0.5359559059143066,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.44140625,
|
|
"rewards/FMTORM/std": 0.1614537090063095,
|
|
"rewards/VQAORM/mean": 0.515625,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 61,
|
|
"train_speed(iter/s)": 0.00605
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2080.0,
|
|
"completions/mean_length": 627.1953125,
|
|
"completions/min_length": 153.0,
|
|
"epoch": 0.035952449985503045,
|
|
"grad_norm": 0.45918061575097496,
|
|
"kl": 0.002593994140625,
|
|
"learning_rate": 2.980769230769231e-07,
|
|
"loss": 2.600867219371139e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.9453125,
|
|
"reward_std": 0.5174567699432373,
|
|
"rewards/CSTORM/mean": 0.1171875,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.4296875,
|
|
"rewards/FMTORM/std": 0.17450013756752014,
|
|
"rewards/VQAORM/mean": 0.3984375,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 62,
|
|
"train_speed(iter/s)": 0.006108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1512.0,
|
|
"completions/mean_length": 615.1015625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.036532328211075674,
|
|
"grad_norm": 0.45204669675506104,
|
|
"kl": 0.0050048828125,
|
|
"learning_rate": 3.0288461538461536e-07,
|
|
"loss": 5.023028279538266e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.00390625,
|
|
"reward_std": 0.44594937562942505,
|
|
"rewards/CSTORM/mean": 0.16015625,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.4140625,
|
|
"rewards/FMTORM/std": 0.1893770843744278,
|
|
"rewards/VQAORM/mean": 0.4296875,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 63,
|
|
"train_speed(iter/s)": 0.005987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2433.0,
|
|
"completions/mean_length": 594.71875,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.0371122064366483,
|
|
"grad_norm": 9.091879932631864,
|
|
"kl": 0.00925445556640625,
|
|
"learning_rate": 3.076923076923077e-07,
|
|
"loss": 9.232923730451148e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.12890625,
|
|
"reward_std": 0.5745972990989685,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.4296875,
|
|
"rewards/FMTORM/std": 0.17450013756752014,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 64,
|
|
"train_speed(iter/s)": 0.005829
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1904.0,
|
|
"completions/mean_length": 620.4453125,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.03769208466222093,
|
|
"grad_norm": 0.45183131529566484,
|
|
"kl": 0.00501251220703125,
|
|
"learning_rate": 3.1249999999999997e-07,
|
|
"loss": 5.013302143197507e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.9921875,
|
|
"reward_std": 0.48497459292411804,
|
|
"rewards/CSTORM/mean": 0.10546875,
|
|
"rewards/CSTORM/std": 0.20478858053684235,
|
|
"rewards/FMTORM/mean": 0.42578125,
|
|
"rewards/FMTORM/std": 0.1784650683403015,
|
|
"rewards/VQAORM/mean": 0.4609375,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 65,
|
|
"train_speed(iter/s)": 0.005885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1833.0,
|
|
"completions/mean_length": 695.734375,
|
|
"completions/min_length": 221.0,
|
|
"epoch": 0.03827196288779356,
|
|
"grad_norm": 0.4700261389856721,
|
|
"kl": 0.0081787109375,
|
|
"learning_rate": 3.1730769230769225e-07,
|
|
"loss": 8.182901183317881e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.77734375,
|
|
"reward_std": 0.4158737361431122,
|
|
"rewards/CSTORM/mean": 0.11328125,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.3671875,
|
|
"rewards/FMTORM/std": 0.22170042991638184,
|
|
"rewards/VQAORM/mean": 0.296875,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 66,
|
|
"train_speed(iter/s)": 0.005851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/mean_length": 614.8203125,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.038851841113366196,
|
|
"grad_norm": 0.46936408693009185,
|
|
"kl": 0.006561279296875,
|
|
"learning_rate": 3.2211538461538464e-07,
|
|
"loss": 6.573856808245182e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.9453125,
|
|
"reward_std": 0.440443217754364,
|
|
"rewards/CSTORM/mean": 0.11328125,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.39453125,
|
|
"rewards/FMTORM/std": 0.20478858053684235,
|
|
"rewards/VQAORM/mean": 0.4375,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 67,
|
|
"train_speed(iter/s)": 0.005744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 6615.0,
|
|
"completions/mean_length": 623.2265625,
|
|
"completions/min_length": 33.0,
|
|
"epoch": 0.039431719338938825,
|
|
"grad_norm": 0.6771261836954758,
|
|
"kl": 0.017486572265625,
|
|
"learning_rate": 3.269230769230769e-07,
|
|
"loss": 1.7510727047920227e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.9375,
|
|
"reward_std": 0.6084057688713074,
|
|
"rewards/CSTORM/mean": 0.12890625,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.41015625,
|
|
"rewards/FMTORM/std": 0.1927177608013153,
|
|
"rewards/VQAORM/mean": 0.3984375,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 68,
|
|
"train_speed(iter/s)": 0.005734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3808.0,
|
|
"completions/mean_length": 613.515625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.040011597564511454,
|
|
"grad_norm": 0.5448403281314412,
|
|
"kl": 0.0122222900390625,
|
|
"learning_rate": 3.317307692307692e-07,
|
|
"loss": 1.2196134775876999e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.984375,
|
|
"reward_std": 0.484527587890625,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.40234375,
|
|
"rewards/FMTORM/std": 0.19899940490722656,
|
|
"rewards/VQAORM/mean": 0.4296875,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 69,
|
|
"train_speed(iter/s)": 0.005754
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1469.0,
|
|
"completions/mean_length": 613.0859375,
|
|
"completions/min_length": 149.0,
|
|
"epoch": 0.04059147579008408,
|
|
"grad_norm": 0.4568742603584753,
|
|
"kl": 0.011383056640625,
|
|
"learning_rate": 3.3653846153846154e-07,
|
|
"loss": 1.1390075087547302e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.14453125,
|
|
"reward_std": 0.614852786064148,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.4296875,
|
|
"rewards/FMTORM/std": 0.17450013756752014,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 70,
|
|
"train_speed(iter/s)": 0.005656
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1375.0,
|
|
"completions/mean_length": 592.0390625,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.04117135401565671,
|
|
"grad_norm": 0.4940102538253269,
|
|
"kl": 0.01531982421875,
|
|
"learning_rate": 3.413461538461538e-07,
|
|
"loss": 1.5334266208810732e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.06640625,
|
|
"reward_std": 0.5734338760375977,
|
|
"rewards/CSTORM/mean": 0.1328125,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.42578125,
|
|
"rewards/FMTORM/std": 0.1784650683403015,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 71,
|
|
"train_speed(iter/s)": 0.00571
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1634.0,
|
|
"completions/mean_length": 622.4609375,
|
|
"completions/min_length": 179.0,
|
|
"epoch": 0.04175123224122934,
|
|
"grad_norm": 0.4333352193583135,
|
|
"kl": 0.015594482421875,
|
|
"learning_rate": 3.461538461538461e-07,
|
|
"loss": 1.5635952877346426e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.11328125,
|
|
"reward_std": 0.5067128539085388,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.43359375,
|
|
"rewards/FMTORM/std": 0.1703527420759201,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 72,
|
|
"train_speed(iter/s)": 0.005621
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2346.0,
|
|
"completions/mean_length": 601.625,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.04233111046680197,
|
|
"grad_norm": 0.5122299628335838,
|
|
"kl": 0.03302001953125,
|
|
"learning_rate": 3.5096153846153844e-07,
|
|
"loss": 3.304751589894295e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.96484375,
|
|
"reward_std": 0.589888334274292,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.390625,
|
|
"rewards/FMTORM/std": 0.20751149952411652,
|
|
"rewards/VQAORM/mean": 0.421875,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 73,
|
|
"train_speed(iter/s)": 0.00566
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1346.0,
|
|
"completions/mean_length": 564.4375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.0429109886923746,
|
|
"grad_norm": 0.5149638992262805,
|
|
"kl": 0.04638671875,
|
|
"learning_rate": 3.557692307692308e-07,
|
|
"loss": 4.6318949898704886e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0234375,
|
|
"reward_std": 0.4839078187942505,
|
|
"rewards/CSTORM/mean": 0.16015625,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.40234375,
|
|
"rewards/FMTORM/std": 0.19899940490722656,
|
|
"rewards/VQAORM/mean": 0.4609375,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 74,
|
|
"train_speed(iter/s)": 0.005575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1708.0,
|
|
"completions/mean_length": 605.671875,
|
|
"completions/min_length": 162.0,
|
|
"epoch": 0.04349086691794723,
|
|
"grad_norm": 0.44092908750291543,
|
|
"kl": 0.0418701171875,
|
|
"learning_rate": 3.6057692307692306e-07,
|
|
"loss": 4.170076863374561e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.00390625,
|
|
"reward_std": 0.5548648834228516,
|
|
"rewards/CSTORM/mean": 0.15625,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.42578125,
|
|
"rewards/FMTORM/std": 0.1784650683403015,
|
|
"rewards/VQAORM/mean": 0.421875,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 75,
|
|
"train_speed(iter/s)": 0.005624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1884.0,
|
|
"completions/mean_length": 604.953125,
|
|
"completions/min_length": 129.0,
|
|
"epoch": 0.044070745143519864,
|
|
"grad_norm": 0.45914798687651365,
|
|
"kl": 0.093994140625,
|
|
"learning_rate": 3.6538461538461534e-07,
|
|
"loss": 9.4098701083567e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.05859375,
|
|
"reward_std": 0.6136692762374878,
|
|
"rewards/CSTORM/mean": 0.16796875,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 76,
|
|
"train_speed(iter/s)": 0.005649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1552.0,
|
|
"completions/mean_length": 596.359375,
|
|
"completions/min_length": 156.0,
|
|
"epoch": 0.04465062336909249,
|
|
"grad_norm": 0.41570206100775225,
|
|
"kl": 0.0634765625,
|
|
"learning_rate": 3.701923076923077e-07,
|
|
"loss": 6.340898835333064e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.10546875,
|
|
"reward_std": 0.4497043490409851,
|
|
"rewards/CSTORM/mean": 0.16015625,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 77,
|
|
"train_speed(iter/s)": 0.005699
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1304.0,
|
|
"completions/mean_length": 602.109375,
|
|
"completions/min_length": 201.0,
|
|
"epoch": 0.04523050159466512,
|
|
"grad_norm": 0.4729404457215273,
|
|
"kl": 0.1229248046875,
|
|
"learning_rate": 3.75e-07,
|
|
"loss": 0.00012297352077439427,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1328125,
|
|
"reward_std": 0.5755809545516968,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.41796875,
|
|
"rewards/FMTORM/std": 0.185893714427948,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 78,
|
|
"train_speed(iter/s)": 0.005749
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1150.0,
|
|
"completions/mean_length": 558.1796875,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.04581037982023775,
|
|
"grad_norm": 0.4787772705064792,
|
|
"kl": 0.109375,
|
|
"learning_rate": 3.798076923076923e-07,
|
|
"loss": 0.0001094197214115411,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0390625,
|
|
"reward_std": 0.4961344301700592,
|
|
"rewards/CSTORM/mean": 0.13671875,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.42578125,
|
|
"rewards/FMTORM/std": 0.1784650683403015,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 79,
|
|
"train_speed(iter/s)": 0.005665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1404.0,
|
|
"completions/mean_length": 592.296875,
|
|
"completions/min_length": 175.0,
|
|
"epoch": 0.04639025804581038,
|
|
"grad_norm": 0.44452121743703005,
|
|
"kl": 0.28955078125,
|
|
"learning_rate": 3.8461538461538463e-07,
|
|
"loss": 0.0002891597105190158,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.16796875,
|
|
"reward_std": 0.476019024848938,
|
|
"rewards/CSTORM/mean": 0.1953125,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.41015625,
|
|
"rewards/FMTORM/std": 0.1927177608013153,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 80,
|
|
"train_speed(iter/s)": 0.005674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1634.0,
|
|
"completions/mean_length": 589.34375,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.04697013627138301,
|
|
"grad_norm": 0.5449991395968717,
|
|
"kl": 0.3056640625,
|
|
"learning_rate": 3.894230769230769e-07,
|
|
"loss": 0.0003056919085793197,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.9140625,
|
|
"reward_std": 0.5063915848731995,
|
|
"rewards/CSTORM/mean": 0.10546875,
|
|
"rewards/CSTORM/std": 0.20478858053684235,
|
|
"rewards/FMTORM/mean": 0.40234375,
|
|
"rewards/FMTORM/std": 0.19899940490722656,
|
|
"rewards/VQAORM/mean": 0.40625,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 81,
|
|
"train_speed(iter/s)": 0.005684
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1765.0,
|
|
"completions/mean_length": 596.171875,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.04755001449695564,
|
|
"grad_norm": 0.4565669248243831,
|
|
"kl": 0.05438232421875,
|
|
"learning_rate": 3.942307692307692e-07,
|
|
"loss": 5.428090298664756e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.23828125,
|
|
"reward_std": 0.555130124092102,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 82,
|
|
"train_speed(iter/s)": 0.005729
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1693.0,
|
|
"completions/mean_length": 568.734375,
|
|
"completions/min_length": 145.0,
|
|
"epoch": 0.04812989272252827,
|
|
"grad_norm": 0.7248294877441634,
|
|
"kl": 0.55615234375,
|
|
"learning_rate": 3.990384615384615e-07,
|
|
"loss": 0.0005550169153138995,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.27734375,
|
|
"reward_std": 0.517459511756897,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4453125,
|
|
"rewards/FMTORM/std": 0.15666775405406952,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 83,
|
|
"train_speed(iter/s)": 0.005774
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9612.0,
|
|
"completions/mean_length": 611.9140625,
|
|
"completions/min_length": 134.0,
|
|
"epoch": 0.048709770948100896,
|
|
"grad_norm": 0.4174774231664859,
|
|
"kl": 0.1376953125,
|
|
"learning_rate": 4.0384615384615386e-07,
|
|
"loss": 0.00013739077257923782,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.25390625,
|
|
"reward_std": 0.4879107177257538,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 84,
|
|
"train_speed(iter/s)": 0.005733
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 4629.0,
|
|
"completions/mean_length": 630.4140625,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.04928964917367353,
|
|
"grad_norm": 0.5450003423002295,
|
|
"kl": 0.431640625,
|
|
"learning_rate": 4.0865384615384614e-07,
|
|
"loss": 0.0004320073639973998,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.07421875,
|
|
"reward_std": 0.505848228931427,
|
|
"rewards/CSTORM/mean": 0.16015625,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 85,
|
|
"train_speed(iter/s)": 0.005654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1198.0,
|
|
"completions/mean_length": 536.921875,
|
|
"completions/min_length": 139.0,
|
|
"epoch": 0.04986952739924616,
|
|
"grad_norm": 0.4515266377181352,
|
|
"kl": 0.139404296875,
|
|
"learning_rate": 4.134615384615384e-07,
|
|
"loss": 0.00013978569768369198,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1171875,
|
|
"reward_std": 0.48495978116989136,
|
|
"rewards/CSTORM/mean": 0.1640625,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.4453125,
|
|
"rewards/FMTORM/std": 0.15666775405406952,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 86,
|
|
"train_speed(iter/s)": 0.00567
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2050.0,
|
|
"completions/mean_length": 573.0390625,
|
|
"completions/min_length": 176.0,
|
|
"epoch": 0.05044940562481879,
|
|
"grad_norm": 0.447574578353933,
|
|
"kl": 0.071044921875,
|
|
"learning_rate": 4.1826923076923076e-07,
|
|
"loss": 7.10177409928292e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.5733010172843933,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.4453125,
|
|
"rewards/FMTORM/std": 0.15666775405406952,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 87,
|
|
"train_speed(iter/s)": 0.005705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1674.0,
|
|
"completions/mean_length": 601.9375,
|
|
"completions/min_length": 141.0,
|
|
"epoch": 0.05102928385039142,
|
|
"grad_norm": 0.418253064825995,
|
|
"kl": 0.20703125,
|
|
"learning_rate": 4.2307692307692304e-07,
|
|
"loss": 0.0002065989247057587,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.06640625,
|
|
"reward_std": 0.4747001528739929,
|
|
"rewards/CSTORM/mean": 0.16015625,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.421875,
|
|
"rewards/FMTORM/std": 0.1822594404220581,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 88,
|
|
"train_speed(iter/s)": 0.005744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1517.0,
|
|
"completions/mean_length": 525.4921875,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.05160916207596405,
|
|
"grad_norm": 0.5181711388426592,
|
|
"kl": 0.0829925537109375,
|
|
"learning_rate": 4.278846153846153e-07,
|
|
"loss": 8.283506031148136e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1328125,
|
|
"reward_std": 0.4145059585571289,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 89,
|
|
"train_speed(iter/s)": 0.005728
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1545.0,
|
|
"completions/mean_length": 581.1640625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.05218904030153668,
|
|
"grad_norm": 0.40513705902474173,
|
|
"kl": 0.1953125,
|
|
"learning_rate": 4.326923076923077e-07,
|
|
"loss": 0.00019524992967490107,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.07421875,
|
|
"reward_std": 0.5940624475479126,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.40625,
|
|
"rewards/FMTORM/std": 0.19592301547527313,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 90,
|
|
"train_speed(iter/s)": 0.005765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3400.0,
|
|
"completions/mean_length": 579.90625,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.052768918527109306,
|
|
"grad_norm": 0.4714252600293889,
|
|
"kl": 0.27783203125,
|
|
"learning_rate": 4.375e-07,
|
|
"loss": 0.00027873553335666656,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1796875,
|
|
"reward_std": 0.53721022605896,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.4140625,
|
|
"rewards/FMTORM/std": 0.1893770843744278,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 91,
|
|
"train_speed(iter/s)": 0.005746
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1143.0,
|
|
"completions/mean_length": 589.015625,
|
|
"completions/min_length": 179.0,
|
|
"epoch": 0.053348796752681935,
|
|
"grad_norm": 0.39635189115664266,
|
|
"kl": 0.0732421875,
|
|
"learning_rate": 4.423076923076923e-07,
|
|
"loss": 7.340466981986538e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.09375,
|
|
"reward_std": 0.4301788806915283,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.4296875,
|
|
"rewards/FMTORM/std": 0.17450013756752014,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 92,
|
|
"train_speed(iter/s)": 0.005792
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1559.0,
|
|
"completions/mean_length": 555.703125,
|
|
"completions/min_length": 48.0,
|
|
"epoch": 0.053928674978254564,
|
|
"grad_norm": 0.5313771406454788,
|
|
"kl": 0.1767578125,
|
|
"learning_rate": 4.471153846153846e-07,
|
|
"loss": 0.00017696780560072511,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.03515625,
|
|
"reward_std": 0.4256575405597687,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.421875,
|
|
"rewards/FMTORM/std": 0.1822594404220581,
|
|
"rewards/VQAORM/mean": 0.4609375,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 93,
|
|
"train_speed(iter/s)": 0.005805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1221.0,
|
|
"completions/mean_length": 545.4375,
|
|
"completions/min_length": 156.0,
|
|
"epoch": 0.0545085532038272,
|
|
"grad_norm": 0.49118331343764293,
|
|
"kl": 0.0560302734375,
|
|
"learning_rate": 4.519230769230769e-07,
|
|
"loss": 5.613732719211839e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3046875,
|
|
"reward_std": 0.4407268762588501,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.47265625,
|
|
"rewards/FMTORM/std": 0.11413132399320602,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 94,
|
|
"train_speed(iter/s)": 0.005851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1904.0,
|
|
"completions/mean_length": 567.5234375,
|
|
"completions/min_length": 184.0,
|
|
"epoch": 0.05508843142939983,
|
|
"grad_norm": 0.4052541109546478,
|
|
"kl": 0.0472412109375,
|
|
"learning_rate": 4.567307692307692e-07,
|
|
"loss": 4.714465103461407e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1015625,
|
|
"reward_std": 0.39611148834228516,
|
|
"rewards/CSTORM/mean": 0.1640625,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.4375,
|
|
"rewards/FMTORM/std": 0.16600920259952545,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 95,
|
|
"train_speed(iter/s)": 0.005882
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 597.9765625,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.05566830965497246,
|
|
"grad_norm": 0.8169080472217776,
|
|
"kl": 0.8134765625,
|
|
"learning_rate": 4.6153846153846156e-07,
|
|
"loss": 0.0008119833655655384,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.07421875,
|
|
"reward_std": 0.5277284383773804,
|
|
"rewards/CSTORM/mean": 0.15625,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.43359375,
|
|
"rewards/FMTORM/std": 0.1703527420759201,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 96,
|
|
"train_speed(iter/s)": 0.005776
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1059.0,
|
|
"completions/mean_length": 539.2421875,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.056248187880545086,
|
|
"grad_norm": 0.43197572371204646,
|
|
"kl": 0.1207275390625,
|
|
"learning_rate": 4.6634615384615384e-07,
|
|
"loss": 0.00012101048196200281,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.375,
|
|
"reward_std": 0.43124261498451233,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 97,
|
|
"train_speed(iter/s)": 0.005821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1872.0,
|
|
"completions/mean_length": 548.4765625,
|
|
"completions/min_length": 144.0,
|
|
"epoch": 0.056828066106117715,
|
|
"grad_norm": 0.4293012039742614,
|
|
"kl": 0.129150390625,
|
|
"learning_rate": 4.711538461538461e-07,
|
|
"loss": 0.0001291538355872035,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0625,
|
|
"reward_std": 0.3406212031841278,
|
|
"rewards/CSTORM/mean": 0.13671875,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.46484375,
|
|
"rewards/FMTORM/std": 0.1283387839794159,
|
|
"rewards/VQAORM/mean": 0.4609375,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 98,
|
|
"train_speed(iter/s)": 0.005858
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1148.0,
|
|
"completions/mean_length": 552.265625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.057407944331690344,
|
|
"grad_norm": 0.45355196957569965,
|
|
"kl": 0.3126220703125,
|
|
"learning_rate": 4.759615384615384e-07,
|
|
"loss": 0.00031138764461502433,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.03125,
|
|
"reward_std": 0.5411145091056824,
|
|
"rewards/CSTORM/mean": 0.13671875,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.44140625,
|
|
"rewards/FMTORM/std": 0.1614537090063095,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 99,
|
|
"train_speed(iter/s)": 0.005901
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 544.953125,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.05798782255726297,
|
|
"grad_norm": 0.4458823183651969,
|
|
"kl": 0.0662841796875,
|
|
"learning_rate": 4.807692307692307e-07,
|
|
"loss": 6.619822670472786e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.21875,
|
|
"reward_std": 0.42830371856689453,
|
|
"rewards/CSTORM/mean": 0.1953125,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 100,
|
|
"train_speed(iter/s)": 0.0058
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3733.0,
|
|
"completions/mean_length": 581.375,
|
|
"completions/min_length": 146.0,
|
|
"epoch": 0.0585677007828356,
|
|
"grad_norm": 0.42493759692608685,
|
|
"kl": 0.0667724609375,
|
|
"learning_rate": 4.855769230769231e-07,
|
|
"loss": 6.661020597675815e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.203125,
|
|
"reward_std": 0.4290696382522583,
|
|
"rewards/CSTORM/mean": 0.1796875,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 101,
|
|
"train_speed(iter/s)": 0.005749
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9288.0,
|
|
"completions/mean_length": 560.09375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.05914757900840823,
|
|
"grad_norm": 0.44262578228905186,
|
|
"kl": 0.00714874267578125,
|
|
"learning_rate": 4.903846153846153e-07,
|
|
"loss": 7.211091087810928e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3359375,
|
|
"reward_std": 0.3655553460121155,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 102,
|
|
"train_speed(iter/s)": 0.005717
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1573.0,
|
|
"completions/mean_length": 568.7265625,
|
|
"completions/min_length": 151.0,
|
|
"epoch": 0.05972745723398087,
|
|
"grad_norm": 0.44233998301645727,
|
|
"kl": 0.02825927734375,
|
|
"learning_rate": 4.951923076923076e-07,
|
|
"loss": 2.825263436534442e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.17578125,
|
|
"reward_std": 0.46998775005340576,
|
|
"rewards/CSTORM/mean": 0.1640625,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.44921875,
|
|
"rewards/FMTORM/std": 0.1516295224428177,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 103,
|
|
"train_speed(iter/s)": 0.005727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1012.0,
|
|
"completions/mean_length": 509.90625,
|
|
"completions/min_length": 134.0,
|
|
"epoch": 0.060307335459553496,
|
|
"grad_norm": 0.40717376529552,
|
|
"kl": 0.25634765625,
|
|
"learning_rate": 5e-07,
|
|
"loss": 0.00025619269581511617,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.140625,
|
|
"reward_std": 0.4821103811264038,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 104,
|
|
"train_speed(iter/s)": 0.005689
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1292.0,
|
|
"completions/mean_length": 551.4375,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.060887213685126125,
|
|
"grad_norm": 0.4448484468870084,
|
|
"kl": 0.07666015625,
|
|
"learning_rate": 4.999999007067563e-07,
|
|
"loss": 7.701865979470313e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.13671875,
|
|
"reward_std": 0.49656254053115845,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.46484375,
|
|
"rewards/FMTORM/std": 0.1283387839794159,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 105,
|
|
"train_speed(iter/s)": 0.005674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1110.0,
|
|
"completions/mean_length": 544.859375,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.061467091910698754,
|
|
"grad_norm": 0.3914304035825463,
|
|
"kl": 0.0428466796875,
|
|
"learning_rate": 4.999996028271129e-07,
|
|
"loss": 4.2791536543518305e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.171875,
|
|
"reward_std": 0.42255425453186035,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 106,
|
|
"train_speed(iter/s)": 0.005714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3585.0,
|
|
"completions/mean_length": 533.0,
|
|
"completions/min_length": 164.0,
|
|
"epoch": 0.06204697013627138,
|
|
"grad_norm": 0.3962999009173884,
|
|
"kl": 0.0311279296875,
|
|
"learning_rate": 4.999991063613326e-07,
|
|
"loss": 3.108434975729324e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.3735966980457306,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4453125,
|
|
"rewards/FMTORM/std": 0.15666775405406952,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 107,
|
|
"train_speed(iter/s)": 0.005652
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9664.0,
|
|
"completions/mean_length": 579.6875,
|
|
"completions/min_length": 123.0,
|
|
"epoch": 0.06262684836184401,
|
|
"grad_norm": 0.39786295528302895,
|
|
"kl": 0.057861328125,
|
|
"learning_rate": 4.999984113098537e-07,
|
|
"loss": 5.7882803957909346e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1953125,
|
|
"reward_std": 0.3768216371536255,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 108,
|
|
"train_speed(iter/s)": 0.005596
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2223.0,
|
|
"completions/mean_length": 543.484375,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.06320672658741665,
|
|
"grad_norm": 0.49101894868819596,
|
|
"kl": 0.020751953125,
|
|
"learning_rate": 4.999975176732895e-07,
|
|
"loss": 2.0790072085219435e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.17578125,
|
|
"reward_std": 0.39500734210014343,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 109,
|
|
"train_speed(iter/s)": 0.005541
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1858.0,
|
|
"completions/mean_length": 596.5234375,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.06378660481298927,
|
|
"grad_norm": 0.41910609908027086,
|
|
"kl": 0.02484130859375,
|
|
"learning_rate": 4.99996425452429e-07,
|
|
"loss": 2.4867651518434286e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1875,
|
|
"reward_std": 0.38529181480407715,
|
|
"rewards/CSTORM/mean": 0.1640625,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 110,
|
|
"train_speed(iter/s)": 0.00555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1418.0,
|
|
"completions/mean_length": 521.6796875,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.0643664830385619,
|
|
"grad_norm": 0.5013405379414168,
|
|
"kl": 0.0982666015625,
|
|
"learning_rate": 4.99995134648236e-07,
|
|
"loss": 9.792902710614726e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.16015625,
|
|
"reward_std": 0.45142030715942383,
|
|
"rewards/CSTORM/mean": 0.16796875,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.4453125,
|
|
"rewards/FMTORM/std": 0.15666775405406952,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 111,
|
|
"train_speed(iter/s)": 0.005582
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1463.0,
|
|
"completions/mean_length": 579.4609375,
|
|
"completions/min_length": 198.0,
|
|
"epoch": 0.06494636126413453,
|
|
"grad_norm": 0.5030443161409431,
|
|
"kl": 0.0220947265625,
|
|
"learning_rate": 4.999936452618499e-07,
|
|
"loss": 2.210121601819992e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.05859375,
|
|
"reward_std": 0.5309361815452576,
|
|
"rewards/CSTORM/mean": 0.125,
|
|
"rewards/CSTORM/std": 0.2173570692539215,
|
|
"rewards/FMTORM/mean": 0.43359375,
|
|
"rewards/FMTORM/std": 0.1703527420759201,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 112,
|
|
"train_speed(iter/s)": 0.005525
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2542.0,
|
|
"completions/mean_length": 545.4765625,
|
|
"completions/min_length": 148.0,
|
|
"epoch": 0.06552623948970716,
|
|
"grad_norm": 0.42547046500218394,
|
|
"kl": 0.093017578125,
|
|
"learning_rate": 4.999919572945851e-07,
|
|
"loss": 9.322773985331878e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.27734375,
|
|
"reward_std": 0.5576099157333374,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4375,
|
|
"rewards/FMTORM/std": 0.16600920259952545,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 113,
|
|
"train_speed(iter/s)": 0.005549
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1347.0,
|
|
"completions/mean_length": 534.265625,
|
|
"completions/min_length": 150.0,
|
|
"epoch": 0.06610611771527979,
|
|
"grad_norm": 0.4131753893801194,
|
|
"kl": 0.015899658203125,
|
|
"learning_rate": 4.999900707479313e-07,
|
|
"loss": 1.5884053937043063e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.33984375,
|
|
"reward_std": 0.4016657769680023,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 114,
|
|
"train_speed(iter/s)": 0.00558
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2173.0,
|
|
"completions/mean_length": 568.03125,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.06668599594085242,
|
|
"grad_norm": 0.4670604000855393,
|
|
"kl": 0.08642578125,
|
|
"learning_rate": 4.999879856235539e-07,
|
|
"loss": 8.663435437483713e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 0.96484375,
|
|
"reward_std": 0.364618718624115,
|
|
"rewards/CSTORM/mean": 0.109375,
|
|
"rewards/CSTORM/std": 0.20751149952411652,
|
|
"rewards/FMTORM/mean": 0.46484375,
|
|
"rewards/FMTORM/std": 0.1283387839794159,
|
|
"rewards/VQAORM/mean": 0.390625,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 115,
|
|
"train_speed(iter/s)": 0.005584
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1629.0,
|
|
"completions/mean_length": 588.078125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.06726587416642506,
|
|
"grad_norm": 0.4001819107891884,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 4.999857019232931e-07,
|
|
"loss": 7.002171332715079e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.06640625,
|
|
"reward_std": 0.3218327760696411,
|
|
"rewards/CSTORM/mean": 0.16796875,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.4453125,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 116,
|
|
"train_speed(iter/s)": 0.005616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1555.0,
|
|
"completions/mean_length": 578.5234375,
|
|
"completions/min_length": 175.0,
|
|
"epoch": 0.06784575239199768,
|
|
"grad_norm": 0.5032568885963471,
|
|
"kl": 0.0340423583984375,
|
|
"learning_rate": 4.999832196491644e-07,
|
|
"loss": 3.400489003979601e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.3749634623527527,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 117,
|
|
"train_speed(iter/s)": 0.005649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2385.0,
|
|
"completions/mean_length": 610.1171875,
|
|
"completions/min_length": 216.0,
|
|
"epoch": 0.06842563061757032,
|
|
"grad_norm": 0.42841258725788406,
|
|
"kl": 0.137298583984375,
|
|
"learning_rate": 4.999805388033589e-07,
|
|
"loss": 0.00013771667727269232,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.234375,
|
|
"reward_std": 0.45900553464889526,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.46484375,
|
|
"rewards/FMTORM/std": 0.1283387839794159,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 118,
|
|
"train_speed(iter/s)": 0.005592
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1604.0,
|
|
"completions/mean_length": 589.8359375,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.06900550884314294,
|
|
"grad_norm": 0.46050509076944723,
|
|
"kl": 0.0552978515625,
|
|
"learning_rate": 4.999776593882425e-07,
|
|
"loss": 5.5482902098447084e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.43400269746780396,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 119,
|
|
"train_speed(iter/s)": 0.00558
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2122.0,
|
|
"completions/mean_length": 550.7890625,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.06958538706871557,
|
|
"grad_norm": 0.4499706050264643,
|
|
"kl": 0.033599853515625,
|
|
"learning_rate": 4.999745814063567e-07,
|
|
"loss": 3.361747076269239e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.234375,
|
|
"reward_std": 0.33087271451950073,
|
|
"rewards/CSTORM/mean": 0.17578125,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.47265625,
|
|
"rewards/FMTORM/std": 0.11413132399320602,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 120,
|
|
"train_speed(iter/s)": 0.00557
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1021.0,
|
|
"completions/mean_length": 521.1640625,
|
|
"completions/min_length": 160.0,
|
|
"epoch": 0.0701652652942882,
|
|
"grad_norm": 0.4748371405138821,
|
|
"kl": 0.0947265625,
|
|
"learning_rate": 4.999713048604183e-07,
|
|
"loss": 9.481500455876812e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1640625,
|
|
"reward_std": 0.4807431697845459,
|
|
"rewards/CSTORM/mean": 0.16015625,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.46484375,
|
|
"rewards/FMTORM/std": 0.1283387839794159,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 121,
|
|
"train_speed(iter/s)": 0.005605
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 962.0,
|
|
"completions/mean_length": 499.8515625,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.07074514351986083,
|
|
"grad_norm": 0.4708722812605725,
|
|
"kl": 0.180908203125,
|
|
"learning_rate": 4.999678297533188e-07,
|
|
"loss": 0.0001801450562197715,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.48579809069633484,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 122,
|
|
"train_speed(iter/s)": 0.005639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1662.0,
|
|
"completions/mean_length": 528.640625,
|
|
"completions/min_length": 133.0,
|
|
"epoch": 0.07132502174543345,
|
|
"grad_norm": 0.4186728802766371,
|
|
"kl": 0.044921875,
|
|
"learning_rate": 4.999641560881257e-07,
|
|
"loss": 4.4808137317886576e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.37482213973999023,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 123,
|
|
"train_speed(iter/s)": 0.005669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1125.0,
|
|
"completions/mean_length": 534.1171875,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.07190489997100609,
|
|
"grad_norm": 0.5017813342520603,
|
|
"kl": 0.18017578125,
|
|
"learning_rate": 4.999602838680814e-07,
|
|
"loss": 0.00018004936282522976,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.0703125,
|
|
"reward_std": 0.4867282807826996,
|
|
"rewards/CSTORM/mean": 0.16015625,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.45703125,
|
|
"rewards/FMTORM/std": 0.1406865119934082,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 124,
|
|
"train_speed(iter/s)": 0.005702
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1600.0,
|
|
"completions/mean_length": 523.140625,
|
|
"completions/min_length": 141.0,
|
|
"epoch": 0.07248477819657873,
|
|
"grad_norm": 0.40833327996819857,
|
|
"kl": 0.17041015625,
|
|
"learning_rate": 4.999562130966032e-07,
|
|
"loss": 0.000170371204148978,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.23828125,
|
|
"reward_std": 0.3820473849773407,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 125,
|
|
"train_speed(iter/s)": 0.005692
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1252.0,
|
|
"completions/mean_length": 510.671875,
|
|
"completions/min_length": 129.0,
|
|
"epoch": 0.07306465642215135,
|
|
"grad_norm": 0.36502406139554266,
|
|
"kl": 0.020416259765625,
|
|
"learning_rate": 4.999519437772845e-07,
|
|
"loss": 2.0423294699867256e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.23046875,
|
|
"reward_std": 0.4158841371536255,
|
|
"rewards/CSTORM/mean": 0.203125,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 126,
|
|
"train_speed(iter/s)": 0.005722
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1162.0,
|
|
"completions/mean_length": 521.328125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.07364453464772398,
|
|
"grad_norm": 0.4381652916753106,
|
|
"kl": 0.2265625,
|
|
"learning_rate": 4.99947475913893e-07,
|
|
"loss": 0.00022669840836897492,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.10546875,
|
|
"reward_std": 0.42148762941360474,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.4609375,
|
|
"rewards/FMTORM/std": 0.1347113400697708,
|
|
"rewards/VQAORM/mean": 0.4609375,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 127,
|
|
"train_speed(iter/s)": 0.005672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1157.0,
|
|
"completions/mean_length": 482.6796875,
|
|
"completions/min_length": 173.0,
|
|
"epoch": 0.0742244128732966,
|
|
"grad_norm": 0.3628190780760324,
|
|
"kl": 0.019256591796875,
|
|
"learning_rate": 4.999428095103722e-07,
|
|
"loss": 1.9210447135264985e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.3050536811351776,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 128,
|
|
"train_speed(iter/s)": 0.005704
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/mean_length": 508.140625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.07480429109886924,
|
|
"grad_norm": 0.43987334578706344,
|
|
"kl": 0.05535888671875,
|
|
"learning_rate": 4.999379445708409e-07,
|
|
"loss": 5.5383279686793685e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3671875,
|
|
"reward_std": 0.3807604908943176,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 129,
|
|
"train_speed(iter/s)": 0.005736
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1072.0,
|
|
"completions/mean_length": 513.796875,
|
|
"completions/min_length": 151.0,
|
|
"epoch": 0.07538416932444186,
|
|
"grad_norm": 0.5262621729807601,
|
|
"kl": 0.0528564453125,
|
|
"learning_rate": 4.999328810995927e-07,
|
|
"loss": 5.2767143642995507e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.20703125,
|
|
"reward_std": 0.49104273319244385,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 130,
|
|
"train_speed(iter/s)": 0.005767
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1333.0,
|
|
"completions/mean_length": 506.859375,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.0759640475500145,
|
|
"grad_norm": 10.177382394599425,
|
|
"kl": 9.71875,
|
|
"learning_rate": 4.999276191010966e-07,
|
|
"loss": 0.009751962497830391,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1953125,
|
|
"reward_std": 0.41045647859573364,
|
|
"rewards/CSTORM/mean": 0.16796875,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.46484375,
|
|
"rewards/FMTORM/std": 0.1283387839794159,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 131,
|
|
"train_speed(iter/s)": 0.00569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1116.0,
|
|
"completions/mean_length": 513.546875,
|
|
"completions/min_length": 156.0,
|
|
"epoch": 0.07654392577558712,
|
|
"grad_norm": 0.4239694444568269,
|
|
"kl": 0.1192779541015625,
|
|
"learning_rate": 4.99922158579997e-07,
|
|
"loss": 0.00011916932271560654,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.18359375,
|
|
"reward_std": 0.3845181167125702,
|
|
"rewards/CSTORM/mean": 0.1796875,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.47265625,
|
|
"rewards/FMTORM/std": 0.11413132399320602,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 132,
|
|
"train_speed(iter/s)": 0.00572
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1341.0,
|
|
"completions/mean_length": 510.4375,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.07712380400115976,
|
|
"grad_norm": 0.4693627410953227,
|
|
"kl": 0.2750244140625,
|
|
"learning_rate": 4.999164995411134e-07,
|
|
"loss": 0.00027460933779366314,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.234375,
|
|
"reward_std": 0.41844063997268677,
|
|
"rewards/CSTORM/mean": 0.203125,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 133,
|
|
"train_speed(iter/s)": 0.00575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1012.0,
|
|
"completions/mean_length": 480.140625,
|
|
"completions/min_length": 123.0,
|
|
"epoch": 0.07770368222673239,
|
|
"grad_norm": 0.48270610500118344,
|
|
"kl": 0.04046630859375,
|
|
"learning_rate": 4.999106419894406e-07,
|
|
"loss": 4.047770198667422e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.4378463327884674,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 134,
|
|
"train_speed(iter/s)": 0.005781
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1051.0,
|
|
"completions/mean_length": 500.4375,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.07828356045230501,
|
|
"grad_norm": 0.4011248615003033,
|
|
"kl": 0.0643310546875,
|
|
"learning_rate": 4.999045859301482e-07,
|
|
"loss": 6.447601481340826e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.16796875,
|
|
"reward_std": 0.34731289744377136,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 135,
|
|
"train_speed(iter/s)": 0.005813
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1351.0,
|
|
"completions/mean_length": 548.0703125,
|
|
"completions/min_length": 193.0,
|
|
"epoch": 0.07886343867787765,
|
|
"grad_norm": 0.4583045637461052,
|
|
"kl": 0.001148223876953125,
|
|
"learning_rate": 4.998983313685817e-07,
|
|
"loss": 1.1543335176611436e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.42908477783203125,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 136,
|
|
"train_speed(iter/s)": 0.005842
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1141.0,
|
|
"completions/mean_length": 491.1796875,
|
|
"completions/min_length": 164.0,
|
|
"epoch": 0.07944331690345027,
|
|
"grad_norm": 0.46491234920438607,
|
|
"kl": 0.158203125,
|
|
"learning_rate": 4.998918783102611e-07,
|
|
"loss": 0.00015840691048651934,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.3897860050201416,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 137,
|
|
"train_speed(iter/s)": 0.005872
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 6453.0,
|
|
"completions/mean_length": 539.5234375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.08002319512902291,
|
|
"grad_norm": 0.4458937924502905,
|
|
"kl": 0.07373046875,
|
|
"learning_rate": 4.99885226760882e-07,
|
|
"loss": 7.370705861831084e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.47876718640327454,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.45703125,
|
|
"rewards/FMTORM/std": 0.1406865119934082,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 138,
|
|
"train_speed(iter/s)": 0.005842
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2013.0,
|
|
"completions/mean_length": 535.859375,
|
|
"completions/min_length": 184.0,
|
|
"epoch": 0.08060307335459553,
|
|
"grad_norm": 0.521427490755238,
|
|
"kl": 0.06549072265625,
|
|
"learning_rate": 4.998783767263153e-07,
|
|
"loss": 6.522532930830494e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.31640625,
|
|
"reward_std": 0.42180800437927246,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.453125,
|
|
"rewards/FMTORM/std": 0.14631295204162598,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 139,
|
|
"train_speed(iter/s)": 0.005864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/mean_length": 502.6328125,
|
|
"completions/min_length": 158.0,
|
|
"epoch": 0.08118295158016817,
|
|
"grad_norm": 0.4439766900028829,
|
|
"kl": 0.1153564453125,
|
|
"learning_rate": 4.998713282126067e-07,
|
|
"loss": 0.00011513941717566922,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.21484375,
|
|
"reward_std": 0.46106860041618347,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 140,
|
|
"train_speed(iter/s)": 0.005813
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1991.0,
|
|
"completions/mean_length": 537.828125,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.08176282980574079,
|
|
"grad_norm": 0.4671467045641364,
|
|
"kl": 0.098876953125,
|
|
"learning_rate": 4.998640812259771e-07,
|
|
"loss": 9.883538587018847e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.2421875,
|
|
"reward_std": 0.40527474880218506,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.4609375,
|
|
"rewards/FMTORM/std": 0.1347113400697708,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 141,
|
|
"train_speed(iter/s)": 0.00582
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1070.0,
|
|
"completions/mean_length": 540.875,
|
|
"completions/min_length": 164.0,
|
|
"epoch": 0.08234270803131342,
|
|
"grad_norm": 0.3816351047797581,
|
|
"kl": 0.08447265625,
|
|
"learning_rate": 4.998566357728231e-07,
|
|
"loss": 8.449151209788397e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.13671875,
|
|
"reward_std": 0.32648321986198425,
|
|
"rewards/CSTORM/mean": 0.15625,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 142,
|
|
"train_speed(iter/s)": 0.005848
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1109.0,
|
|
"completions/mean_length": 508.5625,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.08292258625688606,
|
|
"grad_norm": 0.45253636367983785,
|
|
"kl": 0.084716796875,
|
|
"learning_rate": 4.99848991859716e-07,
|
|
"loss": 8.462686673738062e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3046875,
|
|
"reward_std": 0.3887236714363098,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 143,
|
|
"train_speed(iter/s)": 0.005795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 909.0,
|
|
"completions/mean_length": 510.7734375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.08350246448245868,
|
|
"grad_norm": 0.5374443971707658,
|
|
"kl": 0.198974609375,
|
|
"learning_rate": 4.998411494934021e-07,
|
|
"loss": 0.00019977299962192774,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.46228331327438354,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 144,
|
|
"train_speed(iter/s)": 0.005826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1338.0,
|
|
"completions/mean_length": 554.5078125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.08408234270803132,
|
|
"grad_norm": 0.4560185138138113,
|
|
"kl": 0.0791015625,
|
|
"learning_rate": 4.998331086808035e-07,
|
|
"loss": 7.91518104961142e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.22265625,
|
|
"reward_std": 0.347529798746109,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 145,
|
|
"train_speed(iter/s)": 0.005814
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1228.0,
|
|
"completions/mean_length": 521.8671875,
|
|
"completions/min_length": 167.0,
|
|
"epoch": 0.08466222093360394,
|
|
"grad_norm": 0.4432309881019522,
|
|
"kl": 0.158203125,
|
|
"learning_rate": 4.998248694290168e-07,
|
|
"loss": 0.00015857169637456536,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.32421875,
|
|
"reward_std": 0.48933714628219604,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.47265625,
|
|
"rewards/FMTORM/std": 0.11413132399320602,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 146,
|
|
"train_speed(iter/s)": 0.005826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 6319.0,
|
|
"completions/mean_length": 584.640625,
|
|
"completions/min_length": 145.0,
|
|
"epoch": 0.08524209915917658,
|
|
"grad_norm": 0.46095779187550806,
|
|
"kl": 0.1142578125,
|
|
"learning_rate": 4.998164317453142e-07,
|
|
"loss": 0.00011462393740657717,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1171875,
|
|
"reward_std": 0.46059107780456543,
|
|
"rewards/CSTORM/mean": 0.1640625,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.4609375,
|
|
"rewards/FMTORM/std": 0.1347113400697708,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 147,
|
|
"train_speed(iter/s)": 0.005821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9589.0,
|
|
"completions/mean_length": 589.4375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.0858219773847492,
|
|
"grad_norm": 0.44129312585212327,
|
|
"kl": 0.00159454345703125,
|
|
"learning_rate": 4.998077956371427e-07,
|
|
"loss": 1.5945634004310705e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.33984375,
|
|
"reward_std": 0.37084653973579407,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 148,
|
|
"train_speed(iter/s)": 0.005796
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1186.0,
|
|
"completions/mean_length": 489.8203125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.08640185561032183,
|
|
"grad_norm": 0.4814127481385481,
|
|
"kl": 0.0679931640625,
|
|
"learning_rate": 4.997989611121247e-07,
|
|
"loss": 6.789287726860493e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.17578125,
|
|
"reward_std": 0.4157522916793823,
|
|
"rewards/CSTORM/mean": 0.171875,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 149,
|
|
"train_speed(iter/s)": 0.005825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1450.0,
|
|
"completions/mean_length": 536.7578125,
|
|
"completions/min_length": 140.0,
|
|
"epoch": 0.08698173383589446,
|
|
"grad_norm": 0.5158621430163904,
|
|
"kl": 0.22314453125,
|
|
"learning_rate": 4.997899281780576e-07,
|
|
"loss": 0.0002235053980257362,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.3553577661514282,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 150,
|
|
"train_speed(iter/s)": 0.005846
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1128.0,
|
|
"completions/mean_length": 479.2421875,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.08756161206146709,
|
|
"grad_norm": 0.47870514388728813,
|
|
"kl": 0.00189971923828125,
|
|
"learning_rate": 4.997806968429139e-07,
|
|
"loss": 1.897661377370241e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.32217469811439514,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 151,
|
|
"train_speed(iter/s)": 0.005874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2223.0,
|
|
"completions/mean_length": 518.1875,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.08814149028703973,
|
|
"grad_norm": 0.43777981496872603,
|
|
"kl": 0.06396484375,
|
|
"learning_rate": 4.997712671148413e-07,
|
|
"loss": 6.391964416252449e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.078125,
|
|
"reward_std": 0.4300526976585388,
|
|
"rewards/CSTORM/mean": 0.14453125,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 152,
|
|
"train_speed(iter/s)": 0.005896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1725.0,
|
|
"completions/mean_length": 522.59375,
|
|
"completions/min_length": 153.0,
|
|
"epoch": 0.08872136851261235,
|
|
"grad_norm": 0.41519073852515725,
|
|
"kl": 0.0643310546875,
|
|
"learning_rate": 4.997616390021623e-07,
|
|
"loss": 6.429640052374452e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.2578125,
|
|
"reward_std": 0.4371357262134552,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 153,
|
|
"train_speed(iter/s)": 0.005917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1512.0,
|
|
"completions/mean_length": 494.953125,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.08930124673818499,
|
|
"grad_norm": 0.45938033170732284,
|
|
"kl": 0.0509033203125,
|
|
"learning_rate": 4.99751812513375e-07,
|
|
"loss": 5.0859845941886306e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.15625,
|
|
"reward_std": 0.400809645652771,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 154,
|
|
"train_speed(iter/s)": 0.005942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1383.0,
|
|
"completions/mean_length": 470.2109375,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.08988112496375761,
|
|
"grad_norm": 0.44040973310271003,
|
|
"kl": 0.023036956787109375,
|
|
"learning_rate": 4.997417876571523e-07,
|
|
"loss": 2.3067163056111895e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.255979061126709,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 155,
|
|
"train_speed(iter/s)": 0.005968
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1286.0,
|
|
"completions/mean_length": 520.6328125,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.09046100318933024,
|
|
"grad_norm": 0.477784996638453,
|
|
"kl": 0.0582275390625,
|
|
"learning_rate": 4.997315644423421e-07,
|
|
"loss": 5.832064198330045e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.10546875,
|
|
"reward_std": 0.42085033655166626,
|
|
"rewards/CSTORM/mean": 0.15234375,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.46875,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 156,
|
|
"train_speed(iter/s)": 0.005918
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 925.0,
|
|
"completions/mean_length": 475.7578125,
|
|
"completions/min_length": 163.0,
|
|
"epoch": 0.09104088141490287,
|
|
"grad_norm": 0.5837085036508768,
|
|
"kl": 0.079345703125,
|
|
"learning_rate": 4.997211428779675e-07,
|
|
"loss": 7.951630686875433e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.33984375,
|
|
"reward_std": 0.5007344484329224,
|
|
"rewards/CSTORM/mean": 0.203125,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.47265625,
|
|
"rewards/FMTORM/std": 0.11413132399320602,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 157,
|
|
"train_speed(iter/s)": 0.005946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1648.0,
|
|
"completions/mean_length": 529.7890625,
|
|
"completions/min_length": 128.0,
|
|
"epoch": 0.0916207596404755,
|
|
"grad_norm": 0.44348887197125303,
|
|
"kl": 0.027099609375,
|
|
"learning_rate": 4.997105229732267e-07,
|
|
"loss": 2.7101137675344944e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.3528705835342407,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 158,
|
|
"train_speed(iter/s)": 0.00597
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1051.0,
|
|
"completions/mean_length": 482.5625,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.09220063786604812,
|
|
"grad_norm": 0.4564036171017564,
|
|
"kl": 0.015380859375,
|
|
"learning_rate": 4.996997047374928e-07,
|
|
"loss": 1.5348523447755724e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.5,
|
|
"reward_std": 0.3585315942764282,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 159,
|
|
"train_speed(iter/s)": 0.005998
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1248.0,
|
|
"completions/mean_length": 493.5859375,
|
|
"completions/min_length": 191.0,
|
|
"epoch": 0.09278051609162076,
|
|
"grad_norm": 0.3901237704378526,
|
|
"kl": 0.00450897216796875,
|
|
"learning_rate": 4.996886881803142e-07,
|
|
"loss": 4.510958660830511e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.375,
|
|
"reward_std": 0.30250367522239685,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 160,
|
|
"train_speed(iter/s)": 0.006024
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1410.0,
|
|
"completions/mean_length": 517.703125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.0933603943171934,
|
|
"grad_norm": 0.42349334751242307,
|
|
"kl": 0.0447998046875,
|
|
"learning_rate": 4.99677473311414e-07,
|
|
"loss": 4.477072798181325e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.25390625,
|
|
"reward_std": 0.44190430641174316,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 161,
|
|
"train_speed(iter/s)": 0.006033
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1086.0,
|
|
"completions/mean_length": 490.8359375,
|
|
"completions/min_length": 189.0,
|
|
"epoch": 0.09394027254276602,
|
|
"grad_norm": 0.43523370691405455,
|
|
"kl": 0.045654296875,
|
|
"learning_rate": 4.996660601406907e-07,
|
|
"loss": 4.563013499137014e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.274189829826355,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 162,
|
|
"train_speed(iter/s)": 0.006059
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1009.0,
|
|
"completions/mean_length": 478.8359375,
|
|
"completions/min_length": 144.0,
|
|
"epoch": 0.09452015076833865,
|
|
"grad_norm": 0.4504160069482711,
|
|
"kl": 0.0022125244140625,
|
|
"learning_rate": 4.996544486782174e-07,
|
|
"loss": 2.2043548142391955e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.3791416883468628,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 163,
|
|
"train_speed(iter/s)": 0.006084
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1858.0,
|
|
"completions/mean_length": 508.84375,
|
|
"completions/min_length": 155.0,
|
|
"epoch": 0.09510002899391128,
|
|
"grad_norm": 0.37875163757096053,
|
|
"kl": 0.002410888671875,
|
|
"learning_rate": 4.996426389342428e-07,
|
|
"loss": 2.4040300559136085e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.29795175790786743,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 164,
|
|
"train_speed(iter/s)": 0.006106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1256.0,
|
|
"completions/mean_length": 518.46875,
|
|
"completions/min_length": 129.0,
|
|
"epoch": 0.09567990721948391,
|
|
"grad_norm": 0.5196271337860855,
|
|
"kl": 0.025421142578125,
|
|
"learning_rate": 4.996306309191899e-07,
|
|
"loss": 2.546443465689663e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.3883073031902313,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 165,
|
|
"train_speed(iter/s)": 0.006117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2135.0,
|
|
"completions/mean_length": 545.7578125,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.09625978544505653,
|
|
"grad_norm": 0.5239980075899829,
|
|
"kl": 0.05010986328125,
|
|
"learning_rate": 4.996184246436572e-07,
|
|
"loss": 5.0094495236407965e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.234375,
|
|
"reward_std": 0.4505120813846588,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 166,
|
|
"train_speed(iter/s)": 0.006137
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1214.0,
|
|
"completions/mean_length": 470.5703125,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.09683966367062917,
|
|
"grad_norm": 0.46177452432297383,
|
|
"kl": 0.0543212890625,
|
|
"learning_rate": 4.996060201184183e-07,
|
|
"loss": 5.408241122495383e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.28125,
|
|
"reward_std": 0.3183923065662384,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 167,
|
|
"train_speed(iter/s)": 0.006163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1408.0,
|
|
"completions/mean_length": 491.75,
|
|
"completions/min_length": 123.0,
|
|
"epoch": 0.09741954189620179,
|
|
"grad_norm": 0.4044618372400491,
|
|
"kl": 0.033538818359375,
|
|
"learning_rate": 4.99593417354421e-07,
|
|
"loss": 3.354758155182935e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.31525927782058716,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 168,
|
|
"train_speed(iter/s)": 0.006186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 989.0,
|
|
"completions/mean_length": 502.6953125,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.09799942012177443,
|
|
"grad_norm": 0.5342871441615946,
|
|
"kl": 0.089111328125,
|
|
"learning_rate": 4.995806163627889e-07,
|
|
"loss": 8.909558528102934e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.4857040047645569,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 169,
|
|
"train_speed(iter/s)": 0.006209
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1064.0,
|
|
"completions/mean_length": 479.9453125,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.09857929834734706,
|
|
"grad_norm": 0.3980943011114082,
|
|
"kl": 0.044189453125,
|
|
"learning_rate": 4.995676171548203e-07,
|
|
"loss": 4.420109689817764e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.2954895496368408,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 170,
|
|
"train_speed(iter/s)": 0.006234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 882.0,
|
|
"completions/mean_length": 457.78125,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.09915917657291969,
|
|
"grad_norm": 0.4590179759174827,
|
|
"kl": 0.0195770263671875,
|
|
"learning_rate": 4.995544197419882e-07,
|
|
"loss": 1.9656428776215762e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.3053005337715149,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 171,
|
|
"train_speed(iter/s)": 0.006261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1146.0,
|
|
"completions/mean_length": 474.953125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.09973905479849232,
|
|
"grad_norm": 0.4545171276931075,
|
|
"kl": 0.093505859375,
|
|
"learning_rate": 4.995410241359408e-07,
|
|
"loss": 9.345303988084197e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.33984375,
|
|
"reward_std": 0.3340361714363098,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 172,
|
|
"train_speed(iter/s)": 0.006287
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1063.0,
|
|
"completions/mean_length": 472.2734375,
|
|
"completions/min_length": 189.0,
|
|
"epoch": 0.10031893302406494,
|
|
"grad_norm": 0.45071134190416157,
|
|
"kl": 0.0550537109375,
|
|
"learning_rate": 4.995274303485012e-07,
|
|
"loss": 5.5016804253682494e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.2900753617286682,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 173,
|
|
"train_speed(iter/s)": 0.006256
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1296.0,
|
|
"completions/mean_length": 541.2734375,
|
|
"completions/min_length": 129.0,
|
|
"epoch": 0.10089881124963758,
|
|
"grad_norm": 0.5123572786416843,
|
|
"kl": 0.0712890625,
|
|
"learning_rate": 4.995136383916674e-07,
|
|
"loss": 7.12923938408494e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.4159221053123474,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 174,
|
|
"train_speed(iter/s)": 0.00628
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1322.0,
|
|
"completions/mean_length": 472.09375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.1014786894752102,
|
|
"grad_norm": 0.5106853827963324,
|
|
"kl": 0.12890625,
|
|
"learning_rate": 4.994996482776121e-07,
|
|
"loss": 0.00012838261318393052,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1796875,
|
|
"reward_std": 0.39717555046081543,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.515625,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 175,
|
|
"train_speed(iter/s)": 0.006304
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 4120.0,
|
|
"completions/mean_length": 549.03125,
|
|
"completions/min_length": 139.0,
|
|
"epoch": 0.10205856770078284,
|
|
"grad_norm": 0.413441489912512,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 4.994854600186831e-07,
|
|
"loss": 6.822305294917896e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.203125,
|
|
"reward_std": 0.3765738010406494,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 176,
|
|
"train_speed(iter/s)": 0.006253
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1809.0,
|
|
"completions/mean_length": 488.3046875,
|
|
"completions/min_length": 133.0,
|
|
"epoch": 0.10263844592635546,
|
|
"grad_norm": 0.4422220532835057,
|
|
"kl": 0.00403594970703125,
|
|
"learning_rate": 4.994710736274032e-07,
|
|
"loss": 4.031706794194179e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.3527260422706604,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 177,
|
|
"train_speed(iter/s)": 0.006272
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3738.0,
|
|
"completions/mean_length": 504.046875,
|
|
"completions/min_length": 166.0,
|
|
"epoch": 0.1032183241519281,
|
|
"grad_norm": 0.5320297089716088,
|
|
"kl": 0.160888671875,
|
|
"learning_rate": 4.994564891164699e-07,
|
|
"loss": 0.00016113180026877671,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.1171875,
|
|
"reward_std": 0.4931468665599823,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.46875,
|
|
"rewards/FMTORM/std": 0.12150629609823227,
|
|
"rewards/VQAORM/mean": 0.4609375,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 178,
|
|
"train_speed(iter/s)": 0.00624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1063.0,
|
|
"completions/mean_length": 462.0234375,
|
|
"completions/min_length": 146.0,
|
|
"epoch": 0.10379820237750073,
|
|
"grad_norm": 0.40061339224949855,
|
|
"kl": 0.00494384765625,
|
|
"learning_rate": 4.994417064987554e-07,
|
|
"loss": 4.938045094604604e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.3061651289463043,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 179,
|
|
"train_speed(iter/s)": 0.006246
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1842.0,
|
|
"completions/mean_length": 508.9140625,
|
|
"completions/min_length": 138.0,
|
|
"epoch": 0.10437808060307335,
|
|
"grad_norm": 0.44344185400059677,
|
|
"kl": 0.04155731201171875,
|
|
"learning_rate": 4.99426725787307e-07,
|
|
"loss": 4.13545967603568e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.16015625,
|
|
"reward_std": 0.403787761926651,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 180,
|
|
"train_speed(iter/s)": 0.006266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1676.0,
|
|
"completions/mean_length": 467.9140625,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.10495795882864599,
|
|
"grad_norm": 0.40566041780144135,
|
|
"kl": 0.1162109375,
|
|
"learning_rate": 4.994115469953469e-07,
|
|
"loss": 0.00011633150279521942,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.3245859742164612,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 181,
|
|
"train_speed(iter/s)": 0.006286
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1156.0,
|
|
"completions/mean_length": 464.6328125,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.10553783705421861,
|
|
"grad_norm": 0.4301093797284578,
|
|
"kl": 0.01457977294921875,
|
|
"learning_rate": 4.99396170136272e-07,
|
|
"loss": 1.4633557839260902e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.65234375,
|
|
"reward_std": 0.30438748002052307,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.8203125,
|
|
"rewards/VQAORM/std": 0.3854355216026306,
|
|
"step": 182,
|
|
"train_speed(iter/s)": 0.00631
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1118.0,
|
|
"completions/mean_length": 504.8125,
|
|
"completions/min_length": 145.0,
|
|
"epoch": 0.10611771527979125,
|
|
"grad_norm": 0.4351567374308627,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 4.993805952236539e-07,
|
|
"loss": 6.798197864554822e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.21875,
|
|
"reward_std": 0.3912874758243561,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 183,
|
|
"train_speed(iter/s)": 0.006328
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1281.0,
|
|
"completions/mean_length": 483.4453125,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.10669759350536387,
|
|
"grad_norm": 0.4061198965968709,
|
|
"kl": 0.03662109375,
|
|
"learning_rate": 4.993648222712392e-07,
|
|
"loss": 3.6517805710900575e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.23046875,
|
|
"reward_std": 0.3310391902923584,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 184,
|
|
"train_speed(iter/s)": 0.006351
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1106.0,
|
|
"completions/mean_length": 484.28125,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.1072774717309365,
|
|
"grad_norm": 0.4776300968883395,
|
|
"kl": 0.041259765625,
|
|
"learning_rate": 4.993488512929491e-07,
|
|
"loss": 4.1346582293044776e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.2421875,
|
|
"reward_std": 0.31924110651016235,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 185,
|
|
"train_speed(iter/s)": 0.006361
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1330.0,
|
|
"completions/mean_length": 440.9765625,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.10785734995650913,
|
|
"grad_norm": 0.5058119678140136,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 4.993326823028799e-07,
|
|
"loss": 7.527800335083157e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.25,
|
|
"reward_std": 0.3509744107723236,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.47265625,
|
|
"rewards/FMTORM/std": 0.11413132399320602,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 186,
|
|
"train_speed(iter/s)": 0.006384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1077.0,
|
|
"completions/mean_length": 433.1015625,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.10843722818208176,
|
|
"grad_norm": 0.523180186663441,
|
|
"kl": 0.0036163330078125,
|
|
"learning_rate": 4.993163153153023e-07,
|
|
"loss": 3.6118544812779874e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.28125,
|
|
"reward_std": 0.3749360740184784,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 187,
|
|
"train_speed(iter/s)": 0.006409
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9830.0,
|
|
"completions/mean_length": 533.34375,
|
|
"completions/min_length": 127.0,
|
|
"epoch": 0.1090171064076544,
|
|
"grad_norm": 0.5346086067302998,
|
|
"kl": 0.347412109375,
|
|
"learning_rate": 4.992997503446619e-07,
|
|
"loss": 0.0003472496464382857,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.26953125,
|
|
"reward_std": 0.45557546615600586,
|
|
"rewards/CSTORM/mean": 0.203125,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.47265625,
|
|
"rewards/FMTORM/std": 0.11413132399320602,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 188,
|
|
"train_speed(iter/s)": 0.006361
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1239.0,
|
|
"completions/mean_length": 484.625,
|
|
"completions/min_length": 139.0,
|
|
"epoch": 0.10959698463322702,
|
|
"grad_norm": 0.5761873256411418,
|
|
"kl": 0.00732421875,
|
|
"learning_rate": 4.99282987405579e-07,
|
|
"loss": 7.3402843554504216e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.32421875,
|
|
"reward_std": 0.44120848178863525,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 189,
|
|
"train_speed(iter/s)": 0.006346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1081.0,
|
|
"completions/mean_length": 501.2109375,
|
|
"completions/min_length": 152.0,
|
|
"epoch": 0.11017686285879966,
|
|
"grad_norm": 0.3589618933344457,
|
|
"kl": 0.02679443359375,
|
|
"learning_rate": 4.992660265128489e-07,
|
|
"loss": 2.6859288482228294e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.28515625,
|
|
"reward_std": 0.258328378200531,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 190,
|
|
"train_speed(iter/s)": 0.00637
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1149.0,
|
|
"completions/mean_length": 446.828125,
|
|
"completions/min_length": 161.0,
|
|
"epoch": 0.11075674108437228,
|
|
"grad_norm": 0.5560038181033092,
|
|
"kl": 0.068267822265625,
|
|
"learning_rate": 4.992488676814413e-07,
|
|
"loss": 6.82625686749816e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.15234375,
|
|
"reward_std": 0.4527345597743988,
|
|
"rewards/CSTORM/mean": 0.16796875,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 191,
|
|
"train_speed(iter/s)": 0.006323
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1807.0,
|
|
"completions/mean_length": 509.8515625,
|
|
"completions/min_length": 135.0,
|
|
"epoch": 0.11133661930994491,
|
|
"grad_norm": 0.5185104132727544,
|
|
"kl": 0.0999755859375,
|
|
"learning_rate": 4.992315109265007e-07,
|
|
"loss": 9.983836207538843e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.23046875,
|
|
"reward_std": 0.42146068811416626,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 192,
|
|
"train_speed(iter/s)": 0.006277
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1379.0,
|
|
"completions/mean_length": 450.765625,
|
|
"completions/min_length": 163.0,
|
|
"epoch": 0.11191649753551754,
|
|
"grad_norm": 0.49626216910816706,
|
|
"kl": 0.02532958984375,
|
|
"learning_rate": 4.992139562633462e-07,
|
|
"loss": 2.5227278456441127e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.3771868348121643,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 193,
|
|
"train_speed(iter/s)": 0.006298
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2723.0,
|
|
"completions/mean_length": 469.859375,
|
|
"completions/min_length": 171.0,
|
|
"epoch": 0.11249637576109017,
|
|
"grad_norm": 0.5064749676501888,
|
|
"kl": 0.0069732666015625,
|
|
"learning_rate": 4.991962037074717e-07,
|
|
"loss": 6.987320375628769e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.3350940942764282,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 194,
|
|
"train_speed(iter/s)": 0.006312
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 892.0,
|
|
"completions/mean_length": 433.8671875,
|
|
"completions/min_length": 134.0,
|
|
"epoch": 0.1130762539866628,
|
|
"grad_norm": 0.511940576319788,
|
|
"kl": 0.087799072265625,
|
|
"learning_rate": 4.991782532745457e-07,
|
|
"loss": 8.821644587442279e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.35975906252861023,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 195,
|
|
"train_speed(iter/s)": 0.006337
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1339.0,
|
|
"completions/mean_length": 438.328125,
|
|
"completions/min_length": 138.0,
|
|
"epoch": 0.11365613221223543,
|
|
"grad_norm": 0.570169886749363,
|
|
"kl": 0.0050811767578125,
|
|
"learning_rate": 4.991601049804116e-07,
|
|
"loss": 5.083044470666209e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.21484375,
|
|
"reward_std": 0.443214476108551,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 196,
|
|
"train_speed(iter/s)": 0.006358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 915.0,
|
|
"completions/mean_length": 440.7265625,
|
|
"completions/min_length": 144.0,
|
|
"epoch": 0.11423601043780807,
|
|
"grad_norm": 0.5564538387864988,
|
|
"kl": 0.0057525634765625,
|
|
"learning_rate": 4.991417588410869e-07,
|
|
"loss": 5.751063326897565e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.371845006942749,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 197,
|
|
"train_speed(iter/s)": 0.00638
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1378.0,
|
|
"completions/mean_length": 445.875,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.11481588866338069,
|
|
"grad_norm": 0.5383897150515315,
|
|
"kl": 0.083526611328125,
|
|
"learning_rate": 4.991232148727641e-07,
|
|
"loss": 8.360242645721883e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.296875,
|
|
"reward_std": 0.38557910919189453,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 198,
|
|
"train_speed(iter/s)": 0.006401
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1032.0,
|
|
"completions/mean_length": 454.109375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.11539576688895332,
|
|
"grad_norm": 0.532097813577999,
|
|
"kl": 0.01171875,
|
|
"learning_rate": 4.991044730918103e-07,
|
|
"loss": 1.170951782114571e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.21875,
|
|
"reward_std": 0.3983924686908722,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 199,
|
|
"train_speed(iter/s)": 0.006408
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1318.0,
|
|
"completions/mean_length": 483.234375,
|
|
"completions/min_length": 163.0,
|
|
"epoch": 0.11597564511452595,
|
|
"grad_norm": 0.49361964137544073,
|
|
"kl": 0.02166748046875,
|
|
"learning_rate": 4.990855335147671e-07,
|
|
"loss": 2.163245517294854e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.28125,
|
|
"reward_std": 0.31693893671035767,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 200,
|
|
"train_speed(iter/s)": 0.006427
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1827.0,
|
|
"completions/mean_length": 478.890625,
|
|
"completions/min_length": 126.0,
|
|
"epoch": 0.11655552334009858,
|
|
"grad_norm": 0.43503923708869596,
|
|
"kl": 0.065673828125,
|
|
"learning_rate": 4.990663961583507e-07,
|
|
"loss": 6.579192995559424e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.26953125,
|
|
"reward_std": 0.3722364008426666,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 201,
|
|
"train_speed(iter/s)": 0.00644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 989.0,
|
|
"completions/mean_length": 459.4453125,
|
|
"completions/min_length": 66.0,
|
|
"epoch": 0.1171354015656712,
|
|
"grad_norm": 0.5781743803152606,
|
|
"kl": 0.1427001953125,
|
|
"learning_rate": 4.990470610394519e-07,
|
|
"loss": 0.00014218517753761262,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.2734375,
|
|
"reward_std": 0.3908940553665161,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 202,
|
|
"train_speed(iter/s)": 0.006463
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1000.0,
|
|
"completions/mean_length": 407.2265625,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.11771527979124384,
|
|
"grad_norm": 0.4822286569033236,
|
|
"kl": 0.03369140625,
|
|
"learning_rate": 4.990275281751358e-07,
|
|
"loss": 3.3643558708718047e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.15234375,
|
|
"reward_std": 0.34206727147102356,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 203,
|
|
"train_speed(iter/s)": 0.006418
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 841.0,
|
|
"completions/mean_length": 437.4453125,
|
|
"completions/min_length": 156.0,
|
|
"epoch": 0.11829515801681646,
|
|
"grad_norm": 0.4509423355259062,
|
|
"kl": 0.0049285888671875,
|
|
"learning_rate": 4.990077975826425e-07,
|
|
"loss": 4.928901034872979e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.26480045914649963,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 204,
|
|
"train_speed(iter/s)": 0.006441
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 946.0,
|
|
"completions/mean_length": 454.3203125,
|
|
"completions/min_length": 139.0,
|
|
"epoch": 0.1188750362423891,
|
|
"grad_norm": 0.5124011743602322,
|
|
"kl": 0.082763671875,
|
|
"learning_rate": 4.989878692793861e-07,
|
|
"loss": 8.258216257672757e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.3334563970565796,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 205,
|
|
"train_speed(iter/s)": 0.006463
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1315.0,
|
|
"completions/mean_length": 464.15625,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.11945491446796173,
|
|
"grad_norm": 0.4472399057088549,
|
|
"kl": 0.0503692626953125,
|
|
"learning_rate": 4.989677432829558e-07,
|
|
"loss": 5.0466864195186645e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.2905212640762329,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 206,
|
|
"train_speed(iter/s)": 0.006483
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1440.0,
|
|
"completions/mean_length": 441.7578125,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.12003479269353436,
|
|
"grad_norm": 0.5666557419296947,
|
|
"kl": 0.04364013671875,
|
|
"learning_rate": 4.989474196111146e-07,
|
|
"loss": 4.3440239096526057e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3046875,
|
|
"reward_std": 0.4335278272628784,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 207,
|
|
"train_speed(iter/s)": 0.006504
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1203.0,
|
|
"completions/mean_length": 432.515625,
|
|
"completions/min_length": 123.0,
|
|
"epoch": 0.12061467091910699,
|
|
"grad_norm": 0.5982825386632376,
|
|
"kl": 0.03765869140625,
|
|
"learning_rate": 4.989268982818005e-07,
|
|
"loss": 3.756640217034146e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.4648749530315399,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 208,
|
|
"train_speed(iter/s)": 0.00646
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1595.0,
|
|
"completions/mean_length": 474.1015625,
|
|
"completions/min_length": 139.0,
|
|
"epoch": 0.12119454914467961,
|
|
"grad_norm": 0.46432772324133625,
|
|
"kl": 0.0051422119140625,
|
|
"learning_rate": 4.989061793131256e-07,
|
|
"loss": 5.128305474499939e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.39221763610839844,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 209,
|
|
"train_speed(iter/s)": 0.00648
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1313.0,
|
|
"completions/mean_length": 441.34375,
|
|
"completions/min_length": 137.0,
|
|
"epoch": 0.12177442737025225,
|
|
"grad_norm": 0.5282230879855722,
|
|
"kl": 0.0924072265625,
|
|
"learning_rate": 4.988852627233767e-07,
|
|
"loss": 9.219923231285065e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.17578125,
|
|
"reward_std": 0.39364486932754517,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.4765625,
|
|
"rewards/FMTORM/std": 0.10610081255435944,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 210,
|
|
"train_speed(iter/s)": 0.006501
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 885.0,
|
|
"completions/mean_length": 444.8984375,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.12235430559582487,
|
|
"grad_norm": 0.39139718735606444,
|
|
"kl": 0.0056304931640625,
|
|
"learning_rate": 4.98864148531015e-07,
|
|
"loss": 5.636532478092704e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.22265625,
|
|
"reward_std": 0.29346370697021484,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 211,
|
|
"train_speed(iter/s)": 0.006523
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 879.0,
|
|
"completions/mean_length": 424.8671875,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.12293418382139751,
|
|
"grad_norm": 0.5519738788990808,
|
|
"kl": 0.0269317626953125,
|
|
"learning_rate": 4.988428367546758e-07,
|
|
"loss": 2.697880881896708e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.32594117522239685,
|
|
"rewards/CSTORM/mean": 0.203125,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 212,
|
|
"train_speed(iter/s)": 0.006479
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1344.0,
|
|
"completions/mean_length": 437.265625,
|
|
"completions/min_length": 145.0,
|
|
"epoch": 0.12351406204697013,
|
|
"grad_norm": 0.4865572796994998,
|
|
"kl": 0.0400543212890625,
|
|
"learning_rate": 4.988213274131693e-07,
|
|
"loss": 4.007565803476609e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.2421875,
|
|
"reward_std": 0.39740246534347534,
|
|
"rewards/CSTORM/mean": 0.203125,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 213,
|
|
"train_speed(iter/s)": 0.0065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1721.0,
|
|
"completions/mean_length": 449.03125,
|
|
"completions/min_length": 143.0,
|
|
"epoch": 0.12409394027254277,
|
|
"grad_norm": 0.3934617103622424,
|
|
"kl": 0.0706787109375,
|
|
"learning_rate": 4.987996205254795e-07,
|
|
"loss": 7.091701263561845e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.25,
|
|
"reward_std": 0.28755253553390503,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 214,
|
|
"train_speed(iter/s)": 0.006518
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1224.0,
|
|
"completions/mean_length": 421.984375,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.1246738184981154,
|
|
"grad_norm": 0.4970274092505009,
|
|
"kl": 0.0059661865234375,
|
|
"learning_rate": 4.987777161107654e-07,
|
|
"loss": 5.974321993562626e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3203125,
|
|
"reward_std": 0.33617085218429565,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 215,
|
|
"train_speed(iter/s)": 0.006539
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 864.0,
|
|
"completions/mean_length": 441.9375,
|
|
"completions/min_length": 137.0,
|
|
"epoch": 0.12525369672368802,
|
|
"grad_norm": 0.48968394570106494,
|
|
"kl": 0.025146484375,
|
|
"learning_rate": 4.987556141883595e-07,
|
|
"loss": 2.520570342312567e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.32352396845817566,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 216,
|
|
"train_speed(iter/s)": 0.00656
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1227.0,
|
|
"completions/mean_length": 435.8125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.12583357494926065,
|
|
"grad_norm": 0.5238817588982041,
|
|
"kl": 0.04644775390625,
|
|
"learning_rate": 4.987333147777695e-07,
|
|
"loss": 4.65487755718641e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.363908052444458,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 217,
|
|
"train_speed(iter/s)": 0.006578
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1134.0,
|
|
"completions/mean_length": 415.3203125,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.1264134531748333,
|
|
"grad_norm": 0.4601552202938083,
|
|
"kl": 0.074493408203125,
|
|
"learning_rate": 4.987108178986769e-07,
|
|
"loss": 7.456002640537918e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.3027138113975525,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 218,
|
|
"train_speed(iter/s)": 0.0066
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 469.140625,
|
|
"completions/min_length": 145.0,
|
|
"epoch": 0.12699333140040592,
|
|
"grad_norm": 0.407241526634254,
|
|
"kl": 0.00634765625,
|
|
"learning_rate": 4.986881235709375e-07,
|
|
"loss": 6.3443540057050996e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.2584938406944275,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 219,
|
|
"train_speed(iter/s)": 0.006539
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1056.0,
|
|
"completions/mean_length": 408.203125,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.12757320962597854,
|
|
"grad_norm": 0.5404605488637529,
|
|
"kl": 0.0071258544921875,
|
|
"learning_rate": 4.986652318145815e-07,
|
|
"loss": 7.119101155694807e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.3593299686908722,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 220,
|
|
"train_speed(iter/s)": 0.00656
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1139.0,
|
|
"completions/mean_length": 410.1875,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.12815308785155116,
|
|
"grad_norm": 0.4588673649401949,
|
|
"kl": 0.00653076171875,
|
|
"learning_rate": 4.986421426498135e-07,
|
|
"loss": 6.542299161083065e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.27679967880249023,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 221,
|
|
"train_speed(iter/s)": 0.006582
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 909.0,
|
|
"completions/mean_length": 435.234375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.1287329660771238,
|
|
"grad_norm": 0.46277682862941794,
|
|
"kl": 0.0615081787109375,
|
|
"learning_rate": 4.986188560970118e-07,
|
|
"loss": 6.155042501632124e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.26171875,
|
|
"reward_std": 0.3664497137069702,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 222,
|
|
"train_speed(iter/s)": 0.006603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1091.0,
|
|
"completions/mean_length": 397.65625,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.12931284430269643,
|
|
"grad_norm": 0.5176297703857772,
|
|
"kl": 0.007110595703125,
|
|
"learning_rate": 4.985953721767296e-07,
|
|
"loss": 7.112348157534143e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.3737611472606659,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 223,
|
|
"train_speed(iter/s)": 0.006625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1295.0,
|
|
"completions/mean_length": 428.6015625,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.12989272252826906,
|
|
"grad_norm": 0.453855906188491,
|
|
"kl": 0.0069122314453125,
|
|
"learning_rate": 4.985716909096938e-07,
|
|
"loss": 6.919223778822925e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.23046875,
|
|
"reward_std": 0.23531997203826904,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 224,
|
|
"train_speed(iter/s)": 0.006646
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 981.0,
|
|
"completions/mean_length": 408.0,
|
|
"completions/min_length": 152.0,
|
|
"epoch": 0.1304726007538417,
|
|
"grad_norm": 0.5194106128330653,
|
|
"kl": 0.019805908203125,
|
|
"learning_rate": 4.985478123168057e-07,
|
|
"loss": 1.975904160644859e-05,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.3203125,
|
|
"reward_std": 0.2825453579425812,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 225,
|
|
"train_speed(iter/s)": 0.006667
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1214.0,
|
|
"completions/mean_length": 424.6015625,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.13105247897941433,
|
|
"grad_norm": 0.5303055746587393,
|
|
"kl": 0.131591796875,
|
|
"learning_rate": 4.985237364191406e-07,
|
|
"loss": 0.00013194707571528852,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.15234375,
|
|
"reward_std": 0.3609992265701294,
|
|
"rewards/CSTORM/mean": 0.17578125,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 226,
|
|
"train_speed(iter/s)": 0.006687
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 915.0,
|
|
"completions/mean_length": 416.2109375,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.13163235720498695,
|
|
"grad_norm": 0.5006825672372254,
|
|
"kl": 0.145660400390625,
|
|
"learning_rate": 4.984994632379481e-07,
|
|
"loss": 0.00014483174891211092,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.2421875,
|
|
"reward_std": 0.34320181608200073,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 227,
|
|
"train_speed(iter/s)": 0.006704
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1406.0,
|
|
"completions/mean_length": 461.5234375,
|
|
"completions/min_length": 128.0,
|
|
"epoch": 0.13221223543055957,
|
|
"grad_norm": 0.5346300795346199,
|
|
"kl": 0.00677490234375,
|
|
"learning_rate": 4.984749927946519e-07,
|
|
"loss": 6.777051567041781e-06,
|
|
"memory(GiB)": 51.46,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.42210835218429565,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 228,
|
|
"train_speed(iter/s)": 0.006724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1667.0,
|
|
"completions/mean_length": 458.25,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.13279211365613222,
|
|
"grad_norm": 0.44873908820423175,
|
|
"kl": 0.0496826171875,
|
|
"learning_rate": 4.984503251108498e-07,
|
|
"loss": 4.961837476002984e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.27734375,
|
|
"reward_std": 0.37523338198661804,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 229,
|
|
"train_speed(iter/s)": 0.006741
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 918.0,
|
|
"completions/mean_length": 431.0,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.13337199188170484,
|
|
"grad_norm": 0.4855174320775782,
|
|
"kl": 0.0399169921875,
|
|
"learning_rate": 4.984254602083137e-07,
|
|
"loss": 3.997383828391321e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.37187662720680237,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 230,
|
|
"train_speed(iter/s)": 0.006747
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 877.0,
|
|
"completions/mean_length": 425.59375,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.13395187010727747,
|
|
"grad_norm": 0.5621637313930571,
|
|
"kl": 0.0216064453125,
|
|
"learning_rate": 4.984003981089893e-07,
|
|
"loss": 2.161187876481563e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.25681743025779724,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 231,
|
|
"train_speed(iter/s)": 0.006768
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 883.0,
|
|
"completions/mean_length": 397.8203125,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.13453174833285012,
|
|
"grad_norm": 0.5585062032175748,
|
|
"kl": 0.146087646484375,
|
|
"learning_rate": 4.983751388349968e-07,
|
|
"loss": 0.00014600367285311222,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.28515625,
|
|
"reward_std": 0.3104479908943176,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 232,
|
|
"train_speed(iter/s)": 0.006724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1108.0,
|
|
"completions/mean_length": 412.3515625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.13511162655842274,
|
|
"grad_norm": 0.584954412512775,
|
|
"kl": 0.01336669921875,
|
|
"learning_rate": 4.983496824086302e-07,
|
|
"loss": 1.3397323527897242e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2109375,
|
|
"reward_std": 0.3712383210659027,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 233,
|
|
"train_speed(iter/s)": 0.006745
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 868.0,
|
|
"completions/mean_length": 420.328125,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.13569150478399536,
|
|
"grad_norm": 0.49348030119345593,
|
|
"kl": 0.0155029296875,
|
|
"learning_rate": 4.983240288523573e-07,
|
|
"loss": 1.5521047316724434e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.27438247203826904,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 234,
|
|
"train_speed(iter/s)": 0.006765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 960.0,
|
|
"completions/mean_length": 402.734375,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.13627138300956798,
|
|
"grad_norm": 0.4246524654129109,
|
|
"kl": 0.008056640625,
|
|
"learning_rate": 4.982981781888203e-07,
|
|
"loss": 8.03537022875389e-06,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.30010533332824707,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 235,
|
|
"train_speed(iter/s)": 0.006786
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1045.0,
|
|
"completions/mean_length": 375.5390625,
|
|
"completions/min_length": 47.0,
|
|
"epoch": 0.13685126123514063,
|
|
"grad_norm": 0.6443175857037022,
|
|
"kl": 0.2974853515625,
|
|
"learning_rate": 4.982721304408351e-07,
|
|
"loss": 0.00029782933415845037,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.4677653908729553,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 236,
|
|
"train_speed(iter/s)": 0.006808
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1129.0,
|
|
"completions/mean_length": 417.078125,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.13743113946071325,
|
|
"grad_norm": 0.4451696957316658,
|
|
"kl": 0.01019287109375,
|
|
"learning_rate": 4.982458856313917e-07,
|
|
"loss": 1.01887026175973e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.26182234287261963,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 237,
|
|
"train_speed(iter/s)": 0.006828
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1069.0,
|
|
"completions/mean_length": 399.6640625,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.13801101768628588,
|
|
"grad_norm": 0.5633893822744354,
|
|
"kl": 0.05804443359375,
|
|
"learning_rate": 4.982194437836537e-07,
|
|
"loss": 5.816630437038839e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2890625,
|
|
"reward_std": 0.3610805869102478,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 238,
|
|
"train_speed(iter/s)": 0.006847
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 841.0,
|
|
"completions/mean_length": 421.265625,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.1385908959118585,
|
|
"grad_norm": 0.5082918733861785,
|
|
"kl": 0.062225341796875,
|
|
"learning_rate": 4.981928049209591e-07,
|
|
"loss": 6.221947114681825e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2109375,
|
|
"reward_std": 0.42495468258857727,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 239,
|
|
"train_speed(iter/s)": 0.006867
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1050.0,
|
|
"completions/mean_length": 433.1953125,
|
|
"completions/min_length": 149.0,
|
|
"epoch": 0.13917077413743115,
|
|
"grad_norm": 0.5433610209176927,
|
|
"kl": 0.02130126953125,
|
|
"learning_rate": 4.981659690668194e-07,
|
|
"loss": 2.127188417944126e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.4156363010406494,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 240,
|
|
"train_speed(iter/s)": 0.006887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2535.0,
|
|
"completions/mean_length": 436.7578125,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.13975065236300377,
|
|
"grad_norm": 0.43135969177538497,
|
|
"kl": 0.01788330078125,
|
|
"learning_rate": 4.981389362449203e-07,
|
|
"loss": 1.7875825506052934e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.2963140904903412,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 241,
|
|
"train_speed(iter/s)": 0.006899
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/mean_length": 415.125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.1403305305885764,
|
|
"grad_norm": 0.5670687190247604,
|
|
"kl": 0.07904052734375,
|
|
"learning_rate": 4.981117064791209e-07,
|
|
"loss": 7.894557347754017e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.3209601640701294,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 242,
|
|
"train_speed(iter/s)": 0.006919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1177.0,
|
|
"completions/mean_length": 430.640625,
|
|
"completions/min_length": 55.0,
|
|
"epoch": 0.14091040881414904,
|
|
"grad_norm": 0.4759641590471161,
|
|
"kl": 0.0457763671875,
|
|
"learning_rate": 4.980842797934543e-07,
|
|
"loss": 4.5775195758324116e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.26171875,
|
|
"reward_std": 0.3103329539299011,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 243,
|
|
"train_speed(iter/s)": 0.006921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 860.0,
|
|
"completions/mean_length": 402.25,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.14149028703972166,
|
|
"grad_norm": 0.5015001648117714,
|
|
"kl": 0.033172607421875,
|
|
"learning_rate": 4.980566562121278e-07,
|
|
"loss": 3.3163727493956685e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.23046875,
|
|
"reward_std": 0.30507153272628784,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 244,
|
|
"train_speed(iter/s)": 0.006941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1278.0,
|
|
"completions/mean_length": 406.625,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.14207016526529428,
|
|
"grad_norm": 0.5347823789742122,
|
|
"kl": 0.018157958984375,
|
|
"learning_rate": 4.980288357595221e-07,
|
|
"loss": 1.8130016542272642e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.35010504722595215,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 245,
|
|
"train_speed(iter/s)": 0.00696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1221.0,
|
|
"completions/mean_length": 393.75,
|
|
"completions/min_length": 154.0,
|
|
"epoch": 0.1426500434908669,
|
|
"grad_norm": 0.5830658827920543,
|
|
"kl": 0.044647216796875,
|
|
"learning_rate": 4.980008184601913e-07,
|
|
"loss": 4.475290552363731e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.25,
|
|
"reward_std": 0.45698416233062744,
|
|
"rewards/CSTORM/mean": 0.1953125,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 246,
|
|
"train_speed(iter/s)": 0.00698
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1081.0,
|
|
"completions/mean_length": 420.9765625,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.14322992171643956,
|
|
"grad_norm": 0.5973392458363768,
|
|
"kl": 0.011016845703125,
|
|
"learning_rate": 4.979726043388642e-07,
|
|
"loss": 1.1013069524778984e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.40179964900016785,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 247,
|
|
"train_speed(iter/s)": 0.007
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1406.0,
|
|
"completions/mean_length": 404.4140625,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.14380979994201218,
|
|
"grad_norm": 0.47570665758665287,
|
|
"kl": 0.029510498046875,
|
|
"learning_rate": 4.979441934204426e-07,
|
|
"loss": 2.9470620575011708e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.3515174686908722,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 248,
|
|
"train_speed(iter/s)": 0.007018
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 837.0,
|
|
"completions/mean_length": 404.953125,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.1443896781675848,
|
|
"grad_norm": 0.4502773073290702,
|
|
"kl": 0.055419921875,
|
|
"learning_rate": 4.979155857300019e-07,
|
|
"loss": 5.5375523515976965e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3671875,
|
|
"reward_std": 0.3105451464653015,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 249,
|
|
"train_speed(iter/s)": 0.007037
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1003.0,
|
|
"completions/mean_length": 411.75,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.14496955639315745,
|
|
"grad_norm": 0.4929527156709514,
|
|
"kl": 0.102294921875,
|
|
"learning_rate": 4.978867812927918e-07,
|
|
"loss": 0.00010251591447740793,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.3310580849647522,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 250,
|
|
"train_speed(iter/s)": 0.007058
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1049.0,
|
|
"completions/mean_length": 423.4609375,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.14554943461873007,
|
|
"grad_norm": 0.477044233439131,
|
|
"kl": 0.010284423828125,
|
|
"learning_rate": 4.97857780134235e-07,
|
|
"loss": 1.028154292725958e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.32110679149627686,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 251,
|
|
"train_speed(iter/s)": 0.007078
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2095.0,
|
|
"completions/mean_length": 465.0390625,
|
|
"completions/min_length": 133.0,
|
|
"epoch": 0.1461293128443027,
|
|
"grad_norm": 0.5125221127265864,
|
|
"kl": 0.031768798828125,
|
|
"learning_rate": 4.978285822799283e-07,
|
|
"loss": 3.1851446692598984e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.3480382561683655,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 252,
|
|
"train_speed(iter/s)": 0.00709
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 945.0,
|
|
"completions/mean_length": 388.65625,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.14670919106987532,
|
|
"grad_norm": 0.5111440293119733,
|
|
"kl": 0.017913818359375,
|
|
"learning_rate": 4.977991877556419e-07,
|
|
"loss": 1.7932374248630367e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.25,
|
|
"reward_std": 0.34655123949050903,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 253,
|
|
"train_speed(iter/s)": 0.007095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 732.0,
|
|
"completions/mean_length": 424.53125,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.14728906929544797,
|
|
"grad_norm": 0.46601436722557754,
|
|
"kl": 0.0084228515625,
|
|
"learning_rate": 4.977695965873195e-07,
|
|
"loss": 8.438581062364392e-06,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.325662761926651,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 254,
|
|
"train_speed(iter/s)": 0.007116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1487.0,
|
|
"completions/mean_length": 418.8671875,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.1478689475210206,
|
|
"grad_norm": 0.5746219651143463,
|
|
"kl": 0.010772705078125,
|
|
"learning_rate": 4.977398088010784e-07,
|
|
"loss": 1.0748165550467093e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.29296875,
|
|
"reward_std": 0.26721763610839844,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 255,
|
|
"train_speed(iter/s)": 0.007133
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1144.0,
|
|
"completions/mean_length": 412.4296875,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.1484488257465932,
|
|
"grad_norm": 0.4606030272923643,
|
|
"kl": 0.033843994140625,
|
|
"learning_rate": 4.977098244232098e-07,
|
|
"loss": 3.3861266274470836e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.3352447748184204,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 256,
|
|
"train_speed(iter/s)": 0.007084
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1110.0,
|
|
"completions/mean_length": 430.5078125,
|
|
"completions/min_length": 142.0,
|
|
"epoch": 0.14902870397216583,
|
|
"grad_norm": 0.5565574221042614,
|
|
"kl": 0.009185791015625,
|
|
"learning_rate": 4.976796434801779e-07,
|
|
"loss": 9.199168744089548e-06,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.3484337031841278,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 257,
|
|
"train_speed(iter/s)": 0.007102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1444.0,
|
|
"completions/mean_length": 406.4296875,
|
|
"completions/min_length": 147.0,
|
|
"epoch": 0.14960858219773848,
|
|
"grad_norm": 0.48968420130855017,
|
|
"kl": 0.036865234375,
|
|
"learning_rate": 4.976492659986206e-07,
|
|
"loss": 3.676500637084246e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.140625,
|
|
"reward_std": 0.3281770944595337,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.4609375,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 258,
|
|
"train_speed(iter/s)": 0.007119
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1009.0,
|
|
"completions/mean_length": 378.2734375,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.1501884604233111,
|
|
"grad_norm": 0.577629225829252,
|
|
"kl": 0.0645751953125,
|
|
"learning_rate": 4.976186920053493e-07,
|
|
"loss": 6.453444075305015e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.3971986472606659,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 259,
|
|
"train_speed(iter/s)": 0.007138
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1524.0,
|
|
"completions/mean_length": 439.765625,
|
|
"completions/min_length": 165.0,
|
|
"epoch": 0.15076833864888373,
|
|
"grad_norm": 0.540567899342667,
|
|
"kl": 0.010711669921875,
|
|
"learning_rate": 4.975879215273488e-07,
|
|
"loss": 1.0721716535044834e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.44838809967041016,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 260,
|
|
"train_speed(iter/s)": 0.007155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1515.0,
|
|
"completions/mean_length": 449.25,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.15134821687445638,
|
|
"grad_norm": 0.4511206683955412,
|
|
"kl": 0.009429931640625,
|
|
"learning_rate": 4.975569545917773e-07,
|
|
"loss": 9.447659977013245e-06,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37109375,
|
|
"reward_std": 0.3447628915309906,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 261,
|
|
"train_speed(iter/s)": 0.00717
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1527.0,
|
|
"completions/mean_length": 421.5703125,
|
|
"completions/min_length": 136.0,
|
|
"epoch": 0.151928095100029,
|
|
"grad_norm": 0.4156449824500313,
|
|
"kl": 0.0087890625,
|
|
"learning_rate": 4.975257912259664e-07,
|
|
"loss": 8.793243978288956e-06,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2578125,
|
|
"reward_std": 0.2889266014099121,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 262,
|
|
"train_speed(iter/s)": 0.007186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 939.0,
|
|
"completions/mean_length": 380.8359375,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.15250797332560162,
|
|
"grad_norm": 0.5124387657496847,
|
|
"kl": 0.160888671875,
|
|
"learning_rate": 4.974944314574212e-07,
|
|
"loss": 0.00016059333574958146,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.30846256017684937,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.48046875,
|
|
"rewards/FMTORM/std": 0.0972524881362915,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 263,
|
|
"train_speed(iter/s)": 0.007206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 858.0,
|
|
"completions/mean_length": 418.6953125,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.15308785155117424,
|
|
"grad_norm": 0.5164086454802737,
|
|
"kl": 0.012847900390625,
|
|
"learning_rate": 4.974628753138201e-07,
|
|
"loss": 1.2837423128075898e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.3350940942764282,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 264,
|
|
"train_speed(iter/s)": 0.007226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 760.0,
|
|
"completions/mean_length": 417.8359375,
|
|
"completions/min_length": 154.0,
|
|
"epoch": 0.1536677297767469,
|
|
"grad_norm": 0.5473944089687942,
|
|
"kl": 0.057342529296875,
|
|
"learning_rate": 4.974311228230145e-07,
|
|
"loss": 5.711183257517405e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.2858990430831909,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 265,
|
|
"train_speed(iter/s)": 0.007238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 861.0,
|
|
"completions/mean_length": 389.84375,
|
|
"completions/min_length": 143.0,
|
|
"epoch": 0.15424760800231951,
|
|
"grad_norm": 0.48531836016855623,
|
|
"kl": 0.02227783203125,
|
|
"learning_rate": 4.973991740130294e-07,
|
|
"loss": 2.232869155704975e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.43235695362091064,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 266,
|
|
"train_speed(iter/s)": 0.007257
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 844.0,
|
|
"completions/mean_length": 371.2890625,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.15482748622789214,
|
|
"grad_norm": 0.46943704088375077,
|
|
"kl": 0.0457763671875,
|
|
"learning_rate": 4.973670289120633e-07,
|
|
"loss": 4.584851922118105e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.1484375,
|
|
"reward_std": 0.18318147957324982,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 267,
|
|
"train_speed(iter/s)": 0.00721
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1020.0,
|
|
"completions/mean_length": 388.203125,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.15540736445346479,
|
|
"grad_norm": 0.5691481518805328,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 4.973346875484875e-07,
|
|
"loss": 6.455586844822392e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.28515625,
|
|
"reward_std": 0.37508678436279297,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 268,
|
|
"train_speed(iter/s)": 0.007229
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 901.0,
|
|
"completions/mean_length": 379.9921875,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.1559872426790374,
|
|
"grad_norm": 0.4622334348829097,
|
|
"kl": 0.04486083984375,
|
|
"learning_rate": 4.973021499508467e-07,
|
|
"loss": 4.480893039726652e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.3321160078048706,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 269,
|
|
"train_speed(iter/s)": 0.007248
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/mean_length": 392.1484375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.15656712090461003,
|
|
"grad_norm": 0.5135075814196346,
|
|
"kl": 0.010986328125,
|
|
"learning_rate": 4.972694161478588e-07,
|
|
"loss": 1.0972273230436258e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.40875399112701416,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 270,
|
|
"train_speed(iter/s)": 0.007268
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 850.0,
|
|
"completions/mean_length": 421.40625,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.15714699913018265,
|
|
"grad_norm": 0.4672451138237208,
|
|
"kl": 0.0115966796875,
|
|
"learning_rate": 4.972364861684149e-07,
|
|
"loss": 1.1596182048378978e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.2956213653087616,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 271,
|
|
"train_speed(iter/s)": 0.007287
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1271.0,
|
|
"completions/mean_length": 415.1015625,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.1577268773557553,
|
|
"grad_norm": 0.5672487745116982,
|
|
"kl": 0.013946533203125,
|
|
"learning_rate": 4.972033600415791e-07,
|
|
"loss": 1.3938920346845407e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.20703125,
|
|
"reward_std": 0.40406617522239685,
|
|
"rewards/CSTORM/mean": 0.1875,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 272,
|
|
"train_speed(iter/s)": 0.007305
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1215.0,
|
|
"completions/mean_length": 423.421875,
|
|
"completions/min_length": 135.0,
|
|
"epoch": 0.15830675558132792,
|
|
"grad_norm": 0.44402631871430903,
|
|
"kl": 0.0120849609375,
|
|
"learning_rate": 4.97170037796589e-07,
|
|
"loss": 1.2091581993445288e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2578125,
|
|
"reward_std": 0.3220369815826416,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 273,
|
|
"train_speed(iter/s)": 0.007319
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1014.0,
|
|
"completions/mean_length": 450.75,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.15888663380690055,
|
|
"grad_norm": 0.4791781055189375,
|
|
"kl": 0.010589599609375,
|
|
"learning_rate": 4.971365194628547e-07,
|
|
"loss": 1.057648660207633e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.3863932490348816,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 274,
|
|
"train_speed(iter/s)": 0.007335
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 873.0,
|
|
"completions/mean_length": 387.6328125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.15946651203247317,
|
|
"grad_norm": 0.5155683864459619,
|
|
"kl": 0.028656005859375,
|
|
"learning_rate": 4.9710280506996e-07,
|
|
"loss": 2.854934791685082e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2265625,
|
|
"reward_std": 0.3077860474586487,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 275,
|
|
"train_speed(iter/s)": 0.007353
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/mean_length": 396.2890625,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.16004639025804582,
|
|
"grad_norm": 0.4925095665075886,
|
|
"kl": 0.08953857421875,
|
|
"learning_rate": 4.970688946476613e-07,
|
|
"loss": 8.950887422543019e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.23044784367084503,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 276,
|
|
"train_speed(iter/s)": 0.007371
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 663.0,
|
|
"completions/mean_length": 393.03125,
|
|
"completions/min_length": 133.0,
|
|
"epoch": 0.16062626848361844,
|
|
"grad_norm": 0.4991863000494548,
|
|
"kl": 0.011688232421875,
|
|
"learning_rate": 4.970347882258881e-07,
|
|
"loss": 1.1688092854456045e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.296875,
|
|
"reward_std": 0.32946133613586426,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 277,
|
|
"train_speed(iter/s)": 0.007389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1170.0,
|
|
"completions/mean_length": 404.203125,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.16120614670919106,
|
|
"grad_norm": 0.5026197528445447,
|
|
"kl": 0.011444091796875,
|
|
"learning_rate": 4.970004858347432e-07,
|
|
"loss": 1.1440345588198397e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.19772969186306,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 278,
|
|
"train_speed(iter/s)": 0.007407
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 879.0,
|
|
"completions/mean_length": 358.5625,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.1617860249347637,
|
|
"grad_norm": 0.5931255674788605,
|
|
"kl": 0.0372314453125,
|
|
"learning_rate": 4.969659875045018e-07,
|
|
"loss": 3.7154615711187944e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3515625,
|
|
"reward_std": 0.2654931843280792,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 279,
|
|
"train_speed(iter/s)": 0.007426
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 917.0,
|
|
"completions/mean_length": 354.8046875,
|
|
"completions/min_length": 54.0,
|
|
"epoch": 0.16236590316033633,
|
|
"grad_norm": 0.5496682963602486,
|
|
"kl": 0.030792236328125,
|
|
"learning_rate": 4.969312932656125e-07,
|
|
"loss": 3.083370393142104e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.3440963327884674,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 280,
|
|
"train_speed(iter/s)": 0.007444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 852.0,
|
|
"completions/mean_length": 411.3203125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.16294578138590896,
|
|
"grad_norm": 0.4693170951120438,
|
|
"kl": 0.08380126953125,
|
|
"learning_rate": 4.968964031486968e-07,
|
|
"loss": 8.363881352124736e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.3308112323284149,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 281,
|
|
"train_speed(iter/s)": 0.007464
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 959.0,
|
|
"completions/mean_length": 383.7421875,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.16352565961148158,
|
|
"grad_norm": 0.5192374373075834,
|
|
"kl": 0.01580810546875,
|
|
"learning_rate": 4.968613171845486e-07,
|
|
"loss": 1.5822608474991284e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.345323771238327,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 282,
|
|
"train_speed(iter/s)": 0.007483
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1377.0,
|
|
"completions/mean_length": 410.5625,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.16410553783705423,
|
|
"grad_norm": 0.45871882898441196,
|
|
"kl": 0.012725830078125,
|
|
"learning_rate": 4.968260354041355e-07,
|
|
"loss": 1.2734804840874858e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.2612876296043396,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 283,
|
|
"train_speed(iter/s)": 0.007499
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 910.0,
|
|
"completions/mean_length": 396.7578125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.16468541606262685,
|
|
"grad_norm": 0.4630080218632632,
|
|
"kl": 0.0416259765625,
|
|
"learning_rate": 4.967905578385969e-07,
|
|
"loss": 4.173125125817023e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.25457078218460083,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 284,
|
|
"train_speed(iter/s)": 0.007518
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 995.0,
|
|
"completions/mean_length": 397.828125,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.16526529428819947,
|
|
"grad_norm": 0.474830895833607,
|
|
"kl": 0.011505126953125,
|
|
"learning_rate": 4.96754884519246e-07,
|
|
"loss": 1.1506266673677601e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.2925564646720886,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 285,
|
|
"train_speed(iter/s)": 0.007537
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1090.0,
|
|
"completions/mean_length": 425.203125,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.16584517251377212,
|
|
"grad_norm": 0.3845943629496765,
|
|
"kl": 0.011566162109375,
|
|
"learning_rate": 4.96719015477568e-07,
|
|
"loss": 1.1582413208088838e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.2637236714363098,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 286,
|
|
"train_speed(iter/s)": 0.007552
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1305.0,
|
|
"completions/mean_length": 422.53125,
|
|
"completions/min_length": 153.0,
|
|
"epoch": 0.16642505073934474,
|
|
"grad_norm": 0.5358252184193295,
|
|
"kl": 0.027435302734375,
|
|
"learning_rate": 4.966829507452213e-07,
|
|
"loss": 2.7428410248830914e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.35798126459121704,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 287,
|
|
"train_speed(iter/s)": 0.007565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 661.0,
|
|
"completions/mean_length": 396.3671875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.16700492896491737,
|
|
"grad_norm": 0.5743414170752478,
|
|
"kl": 0.01361083984375,
|
|
"learning_rate": 4.966466903540368e-07,
|
|
"loss": 1.3594117262982763e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.3924960792064667,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 288,
|
|
"train_speed(iter/s)": 0.007585
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 419.5859375,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.16758480719049,
|
|
"grad_norm": 5.9090862098943555,
|
|
"kl": 2.41339111328125,
|
|
"learning_rate": 4.966102343360181e-07,
|
|
"loss": 0.0024242238141596317,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30078125,
|
|
"reward_std": 0.32644224166870117,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 289,
|
|
"train_speed(iter/s)": 0.007519
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1531.0,
|
|
"completions/mean_length": 406.46875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.16816468541606264,
|
|
"grad_norm": 0.5245156675179371,
|
|
"kl": 0.012420654296875,
|
|
"learning_rate": 4.965735827233417e-07,
|
|
"loss": 1.2399785191519186e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.33809107542037964,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 290,
|
|
"train_speed(iter/s)": 0.007535
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 806.0,
|
|
"completions/mean_length": 363.328125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.16874456364163526,
|
|
"grad_norm": 0.5696694865966545,
|
|
"kl": 0.015838623046875,
|
|
"learning_rate": 4.965367355483565e-07,
|
|
"loss": 1.5809660908416845e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.32714977860450745,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 291,
|
|
"train_speed(iter/s)": 0.007554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1195.0,
|
|
"completions/mean_length": 431.2421875,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.16932444186720788,
|
|
"grad_norm": 0.4747196321719814,
|
|
"kl": 0.12646484375,
|
|
"learning_rate": 4.964996928435839e-07,
|
|
"loss": 0.00012616917956620455,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.26953125,
|
|
"reward_std": 0.28461217880249023,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 292,
|
|
"train_speed(iter/s)": 0.007569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1018.0,
|
|
"completions/mean_length": 395.21875,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.1699043200927805,
|
|
"grad_norm": 0.4125190071492477,
|
|
"kl": 0.01336669921875,
|
|
"learning_rate": 4.964624546417183e-07,
|
|
"loss": 1.3378932635532692e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.22507140040397644,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 293,
|
|
"train_speed(iter/s)": 0.007587
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1718.0,
|
|
"completions/mean_length": 419.5078125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.17048419831835315,
|
|
"grad_norm": 0.4531061937807879,
|
|
"kl": 0.022796630859375,
|
|
"learning_rate": 4.96425020975626e-07,
|
|
"loss": 2.2767737391404808e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.32421875,
|
|
"reward_std": 0.24460469186306,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 294,
|
|
"train_speed(iter/s)": 0.007601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1269.0,
|
|
"completions/mean_length": 373.2109375,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.17106407654392577,
|
|
"grad_norm": 0.38139876176601634,
|
|
"kl": 0.035614013671875,
|
|
"learning_rate": 4.963873918783467e-07,
|
|
"loss": 3.572292916942388e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.19660787284374237,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 295,
|
|
"train_speed(iter/s)": 0.007616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2570.0,
|
|
"completions/mean_length": 407.5546875,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.1716439547694984,
|
|
"grad_norm": 0.4618954819948797,
|
|
"kl": 0.012969970703125,
|
|
"learning_rate": 4.963495673830918e-07,
|
|
"loss": 1.2968956980330404e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.3142244815826416,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 296,
|
|
"train_speed(iter/s)": 0.007626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 712.0,
|
|
"completions/mean_length": 381.78125,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.17222383299507105,
|
|
"grad_norm": 0.5097512843189568,
|
|
"kl": 0.015045166015625,
|
|
"learning_rate": 4.963115475232454e-07,
|
|
"loss": 1.5063053069752641e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.25457075238227844,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 297,
|
|
"train_speed(iter/s)": 0.007644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1100.0,
|
|
"completions/mean_length": 403.8984375,
|
|
"completions/min_length": 147.0,
|
|
"epoch": 0.17280371122064367,
|
|
"grad_norm": 0.4824819279724657,
|
|
"kl": 0.014007568359375,
|
|
"learning_rate": 4.962733323323641e-07,
|
|
"loss": 1.4001068848301657e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.609375,
|
|
"reward_std": 0.31707075238227844,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 298,
|
|
"train_speed(iter/s)": 0.007662
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 877.0,
|
|
"completions/mean_length": 439.3125,
|
|
"completions/min_length": 137.0,
|
|
"epoch": 0.1733835894462163,
|
|
"grad_norm": 0.4518674378431594,
|
|
"kl": 0.012298583984375,
|
|
"learning_rate": 4.962349218441771e-07,
|
|
"loss": 1.2297323337406851e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.3233921527862549,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 299,
|
|
"train_speed(iter/s)": 0.007679
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1089.0,
|
|
"completions/mean_length": 358.96875,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.1739634676717889,
|
|
"grad_norm": 0.47128272565251655,
|
|
"kl": 0.015472412109375,
|
|
"learning_rate": 4.961963160925855e-07,
|
|
"loss": 1.5465240721823648e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.26721763610839844,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 300,
|
|
"train_speed(iter/s)": 0.007696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1224.0,
|
|
"completions/mean_length": 398.921875,
|
|
"completions/min_length": 135.0,
|
|
"epoch": 0.17454334589736156,
|
|
"grad_norm": 0.5960608922264999,
|
|
"kl": 0.0146484375,
|
|
"learning_rate": 4.96157515111663e-07,
|
|
"loss": 1.4654877304565161e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.4386635422706604,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 301,
|
|
"train_speed(iter/s)": 0.007706
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 868.0,
|
|
"completions/mean_length": 400.4765625,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.17512322412293418,
|
|
"grad_norm": 0.4629556175398851,
|
|
"kl": 0.011810302734375,
|
|
"learning_rate": 4.961185189356558e-07,
|
|
"loss": 1.1797063052654266e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.333475261926651,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 302,
|
|
"train_speed(iter/s)": 0.007723
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 664.0,
|
|
"completions/mean_length": 353.8828125,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.1757031023485068,
|
|
"grad_norm": 0.5362080639078274,
|
|
"kl": 0.116363525390625,
|
|
"learning_rate": 4.960793275989822e-07,
|
|
"loss": 0.00011617142445174977,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.32730045914649963,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 303,
|
|
"train_speed(iter/s)": 0.007743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/mean_length": 366.265625,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.17628298057407946,
|
|
"grad_norm": 0.5665712249444352,
|
|
"kl": 0.015899658203125,
|
|
"learning_rate": 4.960399411362324e-07,
|
|
"loss": 1.590068313817028e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.3263554573059082,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 304,
|
|
"train_speed(iter/s)": 0.007761
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 779.0,
|
|
"completions/mean_length": 377.9375,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.17686285879965208,
|
|
"grad_norm": 0.574563568731727,
|
|
"kl": 0.013824462890625,
|
|
"learning_rate": 4.960003595821694e-07,
|
|
"loss": 1.3810553355142474e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.26953125,
|
|
"reward_std": 0.4253575801849365,
|
|
"rewards/CSTORM/mean": 0.21484375,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 305,
|
|
"train_speed(iter/s)": 0.00778
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 825.0,
|
|
"completions/mean_length": 401.359375,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.1774427370252247,
|
|
"grad_norm": 0.5636720586373021,
|
|
"kl": 0.2080078125,
|
|
"learning_rate": 4.95960582971728e-07,
|
|
"loss": 0.00020893059263471514,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2890625,
|
|
"reward_std": 0.43746769428253174,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 306,
|
|
"train_speed(iter/s)": 0.007765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1088.0,
|
|
"completions/mean_length": 373.765625,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.17802261525079732,
|
|
"grad_norm": 0.6339533549076737,
|
|
"kl": 0.079833984375,
|
|
"learning_rate": 4.959206113400155e-07,
|
|
"loss": 7.939746865304187e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.38561373949050903,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 307,
|
|
"train_speed(iter/s)": 0.007783
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 693.0,
|
|
"completions/mean_length": 366.296875,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.17860249347636997,
|
|
"grad_norm": 4.815817418293668,
|
|
"kl": 5.2265625,
|
|
"learning_rate": 4.95880444722311e-07,
|
|
"loss": 0.005234010051935911,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2109375,
|
|
"reward_std": 0.3081814646720886,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 308,
|
|
"train_speed(iter/s)": 0.007719
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1344.0,
|
|
"completions/mean_length": 404.3984375,
|
|
"completions/min_length": 136.0,
|
|
"epoch": 0.1791823717019426,
|
|
"grad_norm": 0.49663333818051586,
|
|
"kl": 0.01416015625,
|
|
"learning_rate": 4.95840083154066e-07,
|
|
"loss": 1.4168483176035807e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.32942038774490356,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 309,
|
|
"train_speed(iter/s)": 0.007735
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1161.0,
|
|
"completions/mean_length": 375.15625,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.17976224992751522,
|
|
"grad_norm": 0.4774601378705857,
|
|
"kl": 0.024932861328125,
|
|
"learning_rate": 4.957995266709035e-07,
|
|
"loss": 2.4837343516992405e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.2416265904903412,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 310,
|
|
"train_speed(iter/s)": 0.00775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1123.0,
|
|
"completions/mean_length": 388.3828125,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.18034212815308784,
|
|
"grad_norm": 0.4866649527312828,
|
|
"kl": 0.1505126953125,
|
|
"learning_rate": 4.957587753086193e-07,
|
|
"loss": 0.00015106052160263062,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.3329143524169922,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 311,
|
|
"train_speed(iter/s)": 0.007766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1245.0,
|
|
"completions/mean_length": 382.5703125,
|
|
"completions/min_length": 144.0,
|
|
"epoch": 0.1809220063786605,
|
|
"grad_norm": 0.49277647985374745,
|
|
"kl": 0.01556396484375,
|
|
"learning_rate": 4.957178291031805e-07,
|
|
"loss": 1.5570441973977722e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.23046875,
|
|
"reward_std": 0.37090006470680237,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 312,
|
|
"train_speed(iter/s)": 0.007782
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 701.0,
|
|
"completions/mean_length": 349.2265625,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.1815018846042331,
|
|
"grad_norm": 0.5452218017299357,
|
|
"kl": 0.01849365234375,
|
|
"learning_rate": 4.956766880907269e-07,
|
|
"loss": 1.8467406334821135e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2942620813846588,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 313,
|
|
"train_speed(iter/s)": 0.0078
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1150.0,
|
|
"completions/mean_length": 418.5234375,
|
|
"completions/min_length": 135.0,
|
|
"epoch": 0.18208176282980573,
|
|
"grad_norm": 0.3964968209221986,
|
|
"kl": 0.046875,
|
|
"learning_rate": 4.956353523075694e-07,
|
|
"loss": 4.6879766159690917e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.19601063430309296,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 314,
|
|
"train_speed(iter/s)": 0.007785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1463.0,
|
|
"completions/mean_length": 381.1015625,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.18266164105537838,
|
|
"grad_norm": 0.5046690357761644,
|
|
"kl": 0.0882568359375,
|
|
"learning_rate": 4.955938217901915e-07,
|
|
"loss": 8.786878606770188e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.3951853811740875,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 315,
|
|
"train_speed(iter/s)": 0.007799
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1000.0,
|
|
"completions/mean_length": 373.9609375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.183241519280951,
|
|
"grad_norm": 0.4048395487639116,
|
|
"kl": 0.01519775390625,
|
|
"learning_rate": 4.955520965752482e-07,
|
|
"loss": 1.5212095604510978e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.24852776527404785,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 316,
|
|
"train_speed(iter/s)": 0.007814
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 985.0,
|
|
"completions/mean_length": 353.765625,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.18382139750652363,
|
|
"grad_norm": 0.5439175313053128,
|
|
"kl": 0.0177001953125,
|
|
"learning_rate": 4.955101766995665e-07,
|
|
"loss": 1.7713515262585133e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.2847776412963867,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 317,
|
|
"train_speed(iter/s)": 0.007831
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1094.0,
|
|
"completions/mean_length": 366.4296875,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.18440127573209625,
|
|
"grad_norm": 0.4434909045125833,
|
|
"kl": 0.05169677734375,
|
|
"learning_rate": 4.954680622001452e-07,
|
|
"loss": 5.191365198697895e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.2156400829553604,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 318,
|
|
"train_speed(iter/s)": 0.007835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1029.0,
|
|
"completions/mean_length": 401.125,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.1849811539576689,
|
|
"grad_norm": 0.5372121602139017,
|
|
"kl": 0.03973388671875,
|
|
"learning_rate": 4.954257531141547e-07,
|
|
"loss": 3.971120167989284e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.296875,
|
|
"reward_std": 0.4366565942764282,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 319,
|
|
"train_speed(iter/s)": 0.007836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1601.0,
|
|
"completions/mean_length": 402.4765625,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.18556103218324152,
|
|
"grad_norm": 0.49051356455530787,
|
|
"kl": 0.01666259765625,
|
|
"learning_rate": 4.953832494789375e-07,
|
|
"loss": 1.6661813788232394e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.228829026222229,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 320,
|
|
"train_speed(iter/s)": 0.007847
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 711.0,
|
|
"completions/mean_length": 372.2109375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.18614091040881414,
|
|
"grad_norm": 0.4951955000419579,
|
|
"kl": 0.01446533203125,
|
|
"learning_rate": 4.953405513320073e-07,
|
|
"loss": 1.44725290738279e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.27140435576438904,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 321,
|
|
"train_speed(iter/s)": 0.007864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 813.0,
|
|
"completions/mean_length": 376.59375,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.1867207886343868,
|
|
"grad_norm": 0.5706142567619417,
|
|
"kl": 0.0460205078125,
|
|
"learning_rate": 4.952976587110501e-07,
|
|
"loss": 4.590445678331889e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.35464218258857727,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 322,
|
|
"train_speed(iter/s)": 0.007877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 937.0,
|
|
"completions/mean_length": 366.21875,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.1873006668599594,
|
|
"grad_norm": 0.4707828727978743,
|
|
"kl": 0.025421142578125,
|
|
"learning_rate": 4.952545716539229e-07,
|
|
"loss": 2.5382440071552992e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.24792489409446716,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 323,
|
|
"train_speed(iter/s)": 0.007894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1474.0,
|
|
"completions/mean_length": 382.765625,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.18788054508553204,
|
|
"grad_norm": 0.5369022025253041,
|
|
"kl": 0.0166015625,
|
|
"learning_rate": 4.952112901986547e-07,
|
|
"loss": 1.6581660020165145e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2421875,
|
|
"reward_std": 0.266140878200531,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 324,
|
|
"train_speed(iter/s)": 0.007909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 673.0,
|
|
"completions/mean_length": 387.3359375,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.18846042331110466,
|
|
"grad_norm": 0.47119923825959753,
|
|
"kl": 0.0155029296875,
|
|
"learning_rate": 4.951678143834462e-07,
|
|
"loss": 1.552731373521965e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.27559107542037964,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 325,
|
|
"train_speed(iter/s)": 0.007927
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 832.0,
|
|
"completions/mean_length": 366.90625,
|
|
"completions/min_length": 127.0,
|
|
"epoch": 0.1890403015366773,
|
|
"grad_norm": 0.47001223838378103,
|
|
"kl": 0.01739501953125,
|
|
"learning_rate": 4.951241442466692e-07,
|
|
"loss": 1.7352555005345494e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.27006396651268005,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 326,
|
|
"train_speed(iter/s)": 0.007944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3658.0,
|
|
"completions/mean_length": 440.703125,
|
|
"completions/min_length": 126.0,
|
|
"epoch": 0.18962017976224993,
|
|
"grad_norm": 0.4868149605260058,
|
|
"kl": 0.024200439453125,
|
|
"learning_rate": 4.950802798268673e-07,
|
|
"loss": 2.4139888409990817e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.38250383734703064,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 327,
|
|
"train_speed(iter/s)": 0.007909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 874.0,
|
|
"completions/mean_length": 359.0234375,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.19020005798782255,
|
|
"grad_norm": 0.6682178707728684,
|
|
"kl": 0.016845703125,
|
|
"learning_rate": 4.950362211627556e-07,
|
|
"loss": 1.6876576410140842e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.40583568811416626,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 328,
|
|
"train_speed(iter/s)": 0.007926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1126.0,
|
|
"completions/mean_length": 388.7421875,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.19077993621339517,
|
|
"grad_norm": 0.4157371623430298,
|
|
"kl": 0.02685546875,
|
|
"learning_rate": 4.949919682932206e-07,
|
|
"loss": 2.6840238206204958e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.275722861289978,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 329,
|
|
"train_speed(iter/s)": 0.007941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 880.0,
|
|
"completions/mean_length": 377.84375,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.19135981443896782,
|
|
"grad_norm": 0.44051197782007934,
|
|
"kl": 0.128173828125,
|
|
"learning_rate": 4.949475212573201e-07,
|
|
"loss": 0.0001279540010727942,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.34105029702186584,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 330,
|
|
"train_speed(iter/s)": 0.007892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1022.0,
|
|
"completions/mean_length": 366.125,
|
|
"completions/min_length": 71.0,
|
|
"epoch": 0.19193969266454045,
|
|
"grad_norm": 0.5664883280219482,
|
|
"kl": 0.07366943359375,
|
|
"learning_rate": 4.949028800942833e-07,
|
|
"loss": 7.360964082181454e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3203125,
|
|
"reward_std": 0.38682234287261963,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 331,
|
|
"train_speed(iter/s)": 0.007909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1711.0,
|
|
"completions/mean_length": 406.5859375,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.19251957089011307,
|
|
"grad_norm": 0.48027341552937397,
|
|
"kl": 0.015777587890625,
|
|
"learning_rate": 4.948580448435108e-07,
|
|
"loss": 1.5778958186274394e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.28902068734169006,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 332,
|
|
"train_speed(iter/s)": 0.007921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1071.0,
|
|
"completions/mean_length": 381.8984375,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.19309944911568572,
|
|
"grad_norm": 0.574995281850842,
|
|
"kl": 0.0250244140625,
|
|
"learning_rate": 4.948130155445747e-07,
|
|
"loss": 2.4992988983285613e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.3729668855667114,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 333,
|
|
"train_speed(iter/s)": 0.007937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 916.0,
|
|
"completions/mean_length": 369.5546875,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.19367932734125834,
|
|
"grad_norm": 0.5067646178512729,
|
|
"kl": 0.015167236328125,
|
|
"learning_rate": 4.947677922372179e-07,
|
|
"loss": 1.515787334938068e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.38738328218460083,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 334,
|
|
"train_speed(iter/s)": 0.007952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1431.0,
|
|
"completions/mean_length": 396.984375,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.19425920556683096,
|
|
"grad_norm": 0.48254049126178367,
|
|
"kl": 0.015869140625,
|
|
"learning_rate": 4.947223749613548e-07,
|
|
"loss": 1.5868907212279737e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.2677408456802368,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 335,
|
|
"train_speed(iter/s)": 0.007966
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1216.0,
|
|
"completions/mean_length": 373.5,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.19483908379240358,
|
|
"grad_norm": 0.4983639149710233,
|
|
"kl": 0.015777587890625,
|
|
"learning_rate": 4.946767637570713e-07,
|
|
"loss": 1.5781804904690944e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.36271828413009644,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 336,
|
|
"train_speed(iter/s)": 0.007981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1409.0,
|
|
"completions/mean_length": 379.4765625,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.19541896201797623,
|
|
"grad_norm": 0.5268961255926545,
|
|
"kl": 0.03485107421875,
|
|
"learning_rate": 4.946309586646237e-07,
|
|
"loss": 3.473935430520214e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.3450790345668793,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 337,
|
|
"train_speed(iter/s)": 0.007996
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1291.0,
|
|
"completions/mean_length": 404.2265625,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.19599884024354886,
|
|
"grad_norm": 0.573734537975924,
|
|
"kl": 0.01416015625,
|
|
"learning_rate": 4.945849597244402e-07,
|
|
"loss": 1.414609323546756e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.2769314646720886,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 338,
|
|
"train_speed(iter/s)": 0.008011
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1227.0,
|
|
"completions/mean_length": 361.6875,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.19657871846912148,
|
|
"grad_norm": 0.4949333435606716,
|
|
"kl": 0.01605224609375,
|
|
"learning_rate": 4.945387669771196e-07,
|
|
"loss": 1.608176899026148e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.2588704228401184,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 339,
|
|
"train_speed(iter/s)": 0.008026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 758.0,
|
|
"completions/mean_length": 364.1484375,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.19715859669469413,
|
|
"grad_norm": 0.5241577671055953,
|
|
"kl": 0.0174560546875,
|
|
"learning_rate": 4.94492380463432e-07,
|
|
"loss": 1.7466576537117362e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.3057454228401184,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 340,
|
|
"train_speed(iter/s)": 0.008044
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 837.0,
|
|
"completions/mean_length": 363.0078125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.19773847492026675,
|
|
"grad_norm": 0.44417339733557526,
|
|
"kl": 0.018798828125,
|
|
"learning_rate": 4.944458002243186e-07,
|
|
"loss": 1.884964331111405e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.32473260164260864,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 341,
|
|
"train_speed(iter/s)": 0.00806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1480.0,
|
|
"completions/mean_length": 388.25,
|
|
"completions/min_length": 136.0,
|
|
"epoch": 0.19831835314583937,
|
|
"grad_norm": 0.5108815398293771,
|
|
"kl": 0.01690673828125,
|
|
"learning_rate": 4.94399026300891e-07,
|
|
"loss": 1.6915924788918346e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.2746608853340149,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 342,
|
|
"train_speed(iter/s)": 0.008074
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1170.0,
|
|
"completions/mean_length": 373.65625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.198898231371412,
|
|
"grad_norm": 0.4945663436931116,
|
|
"kl": 0.028564453125,
|
|
"learning_rate": 4.943520587344325e-07,
|
|
"loss": 2.8493019271991216e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.3923494815826416,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 343,
|
|
"train_speed(iter/s)": 0.008089
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 751.0,
|
|
"completions/mean_length": 381.15625,
|
|
"completions/min_length": 137.0,
|
|
"epoch": 0.19947810959698464,
|
|
"grad_norm": 0.48470160455956457,
|
|
"kl": 0.01751708984375,
|
|
"learning_rate": 4.943048975663971e-07,
|
|
"loss": 1.7535663573653437e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.34050828218460083,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 344,
|
|
"train_speed(iter/s)": 0.008107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 890.0,
|
|
"completions/mean_length": 381.0546875,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.20005798782255726,
|
|
"grad_norm": 0.5072611148169265,
|
|
"kl": 0.13671875,
|
|
"learning_rate": 4.942575428384092e-07,
|
|
"loss": 0.00013725551252719015,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2932041585445404,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 345,
|
|
"train_speed(iter/s)": 0.008122
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1553.0,
|
|
"completions/mean_length": 409.5546875,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.2006378660481299,
|
|
"grad_norm": 0.484368018498328,
|
|
"kl": 0.2568359375,
|
|
"learning_rate": 4.942099945922646e-07,
|
|
"loss": 0.00025738083058968186,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.33632156252861023,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 346,
|
|
"train_speed(iter/s)": 0.008135
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 706.0,
|
|
"completions/mean_length": 376.1953125,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.2012177442737025,
|
|
"grad_norm": 0.47469623819647266,
|
|
"kl": 0.03057861328125,
|
|
"learning_rate": 4.941622528699298e-07,
|
|
"loss": 3.054571425309405e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.3323171138763428,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 347,
|
|
"train_speed(iter/s)": 0.008151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1100.0,
|
|
"completions/mean_length": 382.234375,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.20179762249927516,
|
|
"grad_norm": 0.5026985548489377,
|
|
"kl": 0.0447998046875,
|
|
"learning_rate": 4.941143177135417e-07,
|
|
"loss": 4.4765321945305914e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.3579706847667694,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 348,
|
|
"train_speed(iter/s)": 0.008166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 918.0,
|
|
"completions/mean_length": 360.9375,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.20237750072484778,
|
|
"grad_norm": 0.4947726798913088,
|
|
"kl": 0.015838623046875,
|
|
"learning_rate": 4.940661891654084e-07,
|
|
"loss": 1.5833549696253613e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.265625,
|
|
"reward_std": 0.32095611095428467,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 349,
|
|
"train_speed(iter/s)": 0.008183
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1855.0,
|
|
"completions/mean_length": 383.609375,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.2029573789504204,
|
|
"grad_norm": 790.1277507653361,
|
|
"kl": 600.0086669921875,
|
|
"learning_rate": 4.940178672680083e-07,
|
|
"loss": 0.6015796065330505,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.17578125,
|
|
"reward_std": 0.2300376147031784,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.46875,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 350,
|
|
"train_speed(iter/s)": 0.008118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1034.0,
|
|
"completions/mean_length": 374.5078125,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.20353725717599305,
|
|
"grad_norm": 0.5115416320214078,
|
|
"kl": 0.0947265625,
|
|
"learning_rate": 4.939693520639907e-07,
|
|
"loss": 9.456592670176178e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.40166786313056946,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 351,
|
|
"train_speed(iter/s)": 0.008134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 738.0,
|
|
"completions/mean_length": 367.5390625,
|
|
"completions/min_length": 141.0,
|
|
"epoch": 0.20411713540156567,
|
|
"grad_norm": 0.5444788281108334,
|
|
"kl": 0.01611328125,
|
|
"learning_rate": 4.939206435961755e-07,
|
|
"loss": 1.609974424354732e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.369454026222229,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 352,
|
|
"train_speed(iter/s)": 0.00815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1884.0,
|
|
"completions/mean_length": 411.9921875,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.2046970136271383,
|
|
"grad_norm": 0.5109398860366179,
|
|
"kl": 0.0296630859375,
|
|
"learning_rate": 4.938717419075532e-07,
|
|
"loss": 2.969341767311562e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.3532680869102478,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 353,
|
|
"train_speed(iter/s)": 0.008162
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1728.0,
|
|
"completions/mean_length": 372.171875,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.20527689185271092,
|
|
"grad_norm": 0.45983165105209367,
|
|
"kl": 0.0184326171875,
|
|
"learning_rate": 4.938226470412844e-07,
|
|
"loss": 1.8430708223604597e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.2778932452201843,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 354,
|
|
"train_speed(iter/s)": 0.008175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1034.0,
|
|
"completions/mean_length": 384.90625,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.20585677007828357,
|
|
"grad_norm": 0.5700901346674667,
|
|
"kl": 0.0166015625,
|
|
"learning_rate": 4.93773359040701e-07,
|
|
"loss": 1.65914862009231e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.36031997203826904,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 355,
|
|
"train_speed(iter/s)": 0.008189
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1432.0,
|
|
"completions/mean_length": 413.0546875,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.2064366483038562,
|
|
"grad_norm": 0.5369356977293432,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.937238779493046e-07,
|
|
"loss": 6.914993718964979e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.35946178436279297,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 356,
|
|
"train_speed(iter/s)": 0.008202
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 783.0,
|
|
"completions/mean_length": 371.0078125,
|
|
"completions/min_length": 138.0,
|
|
"epoch": 0.2070165265294288,
|
|
"grad_norm": 0.48377522392837224,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 4.936742038107676e-07,
|
|
"loss": 7.068288687150925e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.3328796625137329,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 357,
|
|
"train_speed(iter/s)": 0.008219
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 817.0,
|
|
"completions/mean_length": 320.3359375,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.20759640475500146,
|
|
"grad_norm": 0.5904661172152283,
|
|
"kl": 0.0198974609375,
|
|
"learning_rate": 4.936243366689327e-07,
|
|
"loss": 1.9860404790961184e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.33269578218460083,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 358,
|
|
"train_speed(iter/s)": 0.008236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1294.0,
|
|
"completions/mean_length": 419.6171875,
|
|
"completions/min_length": 141.0,
|
|
"epoch": 0.20817628298057408,
|
|
"grad_norm": 0.4297828593377552,
|
|
"kl": 0.014892578125,
|
|
"learning_rate": 4.935742765678132e-07,
|
|
"loss": 1.4863633623463102e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.23325318098068237,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 359,
|
|
"train_speed(iter/s)": 0.00825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 713.0,
|
|
"completions/mean_length": 369.5078125,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.2087561612061467,
|
|
"grad_norm": 0.5994534635053996,
|
|
"kl": 0.01763916015625,
|
|
"learning_rate": 4.935240235515922e-07,
|
|
"loss": 1.7644913896219805e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.392910361289978,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 360,
|
|
"train_speed(iter/s)": 0.008267
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 744.0,
|
|
"completions/mean_length": 382.84375,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.20933603943171933,
|
|
"grad_norm": 0.40413558465254273,
|
|
"kl": 0.029083251953125,
|
|
"learning_rate": 4.934735776646234e-07,
|
|
"loss": 2.9070664822938852e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59765625,
|
|
"reward_std": 0.30469292402267456,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 361,
|
|
"train_speed(iter/s)": 0.008284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 813.0,
|
|
"completions/mean_length": 360.890625,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.20991591765729198,
|
|
"grad_norm": 0.4532661129286652,
|
|
"kl": 0.01776123046875,
|
|
"learning_rate": 4.934229389514308e-07,
|
|
"loss": 1.7756381566869095e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3359375,
|
|
"reward_std": 0.2860991954803467,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 362,
|
|
"train_speed(iter/s)": 0.0083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1251.0,
|
|
"completions/mean_length": 402.9765625,
|
|
"completions/min_length": 142.0,
|
|
"epoch": 0.2104957958828646,
|
|
"grad_norm": 0.43146072582119277,
|
|
"kl": 0.01763916015625,
|
|
"learning_rate": 4.933721074567084e-07,
|
|
"loss": 1.7615464457776397e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.2614383101463318,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 363,
|
|
"train_speed(iter/s)": 0.008315
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 936.0,
|
|
"completions/mean_length": 346.046875,
|
|
"completions/min_length": 146.0,
|
|
"epoch": 0.21107567410843722,
|
|
"grad_norm": 0.5130541670512665,
|
|
"kl": 0.01953125,
|
|
"learning_rate": 4.933210832253203e-07,
|
|
"loss": 1.9553346646716818e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.25388532876968384,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 364,
|
|
"train_speed(iter/s)": 0.008331
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 953.0,
|
|
"completions/mean_length": 378.5234375,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.21165555233400984,
|
|
"grad_norm": 0.47148727729211226,
|
|
"kl": 0.06317138671875,
|
|
"learning_rate": 4.932698663023009e-07,
|
|
"loss": 6.331621261779219e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.1015625,
|
|
"reward_std": 0.3088291585445404,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.421875,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 365,
|
|
"train_speed(iter/s)": 0.008347
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1288.0,
|
|
"completions/mean_length": 387.9296875,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.2122354305595825,
|
|
"grad_norm": 0.49890304116330175,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 4.932184567328547e-07,
|
|
"loss": 6.763188866898417e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.284743994474411,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 366,
|
|
"train_speed(iter/s)": 0.00836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 761.0,
|
|
"completions/mean_length": 387.3203125,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.21281530878515512,
|
|
"grad_norm": 0.5458708180415159,
|
|
"kl": 0.0167236328125,
|
|
"learning_rate": 4.931668545623561e-07,
|
|
"loss": 1.6689122276147828e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.34775984287261963,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 367,
|
|
"train_speed(iter/s)": 0.008377
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 752.0,
|
|
"completions/mean_length": 383.875,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.21339518701072774,
|
|
"grad_norm": 0.5405419229466605,
|
|
"kl": 0.019775390625,
|
|
"learning_rate": 4.931150598363493e-07,
|
|
"loss": 1.975836858036928e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.41807234287261963,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 368,
|
|
"train_speed(iter/s)": 0.008393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1451.0,
|
|
"completions/mean_length": 382.9609375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.2139750652363004,
|
|
"grad_norm": 0.4833321859568507,
|
|
"kl": 0.02508544921875,
|
|
"learning_rate": 4.930630726005489e-07,
|
|
"loss": 2.497348395991139e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.24568146467208862,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 369,
|
|
"train_speed(iter/s)": 0.008407
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1088.0,
|
|
"completions/mean_length": 384.84375,
|
|
"completions/min_length": 156.0,
|
|
"epoch": 0.214554943461873,
|
|
"grad_norm": 0.49642407388823984,
|
|
"kl": 0.1485595703125,
|
|
"learning_rate": 4.930108929008391e-07,
|
|
"loss": 0.00014827121049165726,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.26171875,
|
|
"reward_std": 0.2704553008079529,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 370,
|
|
"train_speed(iter/s)": 0.008422
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 705.0,
|
|
"completions/mean_length": 368.2578125,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.21513482168744563,
|
|
"grad_norm": 0.519808426787481,
|
|
"kl": 0.01690673828125,
|
|
"learning_rate": 4.929585207832741e-07,
|
|
"loss": 1.6871283150976524e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.3112275004386902,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 371,
|
|
"train_speed(iter/s)": 0.008438
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 901.0,
|
|
"completions/mean_length": 364.65625,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.21571469991301825,
|
|
"grad_norm": 0.4954271382941723,
|
|
"kl": 0.02117919921875,
|
|
"learning_rate": 4.929059562940777e-07,
|
|
"loss": 2.1186222511460073e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.3119390904903412,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 372,
|
|
"train_speed(iter/s)": 0.008453
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/mean_length": 398.9453125,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.2162945781385909,
|
|
"grad_norm": 0.46311175160713863,
|
|
"kl": 0.015411376953125,
|
|
"learning_rate": 4.92853199479644e-07,
|
|
"loss": 1.5420821000589058e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3359375,
|
|
"reward_std": 0.2638554871082306,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 373,
|
|
"train_speed(iter/s)": 0.008468
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1314.0,
|
|
"completions/mean_length": 356.3046875,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.21687445636416353,
|
|
"grad_norm": 0.5122892768150037,
|
|
"kl": 0.02398681640625,
|
|
"learning_rate": 4.928002503865361e-07,
|
|
"loss": 2.3969034373294562e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.31465357542037964,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 374,
|
|
"train_speed(iter/s)": 0.008481
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 661.0,
|
|
"completions/mean_length": 357.1640625,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.21745433458973615,
|
|
"grad_norm": 0.5270757670949199,
|
|
"kl": 0.021484375,
|
|
"learning_rate": 4.927471090614876e-07,
|
|
"loss": 2.1495088731171563e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.30598288774490356,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 375,
|
|
"train_speed(iter/s)": 0.008498
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1305.0,
|
|
"completions/mean_length": 371.7734375,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.2180342128153088,
|
|
"grad_norm": 0.5034716582442482,
|
|
"kl": 0.0184326171875,
|
|
"learning_rate": 4.926937755514011e-07,
|
|
"loss": 1.8387614545645192e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.23828125,
|
|
"reward_std": 0.24766957759857178,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.515625,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 376,
|
|
"train_speed(iter/s)": 0.008511
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 925.0,
|
|
"completions/mean_length": 378.6640625,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.21861409104088142,
|
|
"grad_norm": 0.5845172643131148,
|
|
"kl": 0.0185546875,
|
|
"learning_rate": 4.926402499033491e-07,
|
|
"loss": 1.8519192963140085e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.3888104557991028,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 377,
|
|
"train_speed(iter/s)": 0.008526
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 873.0,
|
|
"completions/mean_length": 367.4609375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.21919396926645404,
|
|
"grad_norm": 0.4708083012055093,
|
|
"kl": 0.0191650390625,
|
|
"learning_rate": 4.925865321645741e-07,
|
|
"loss": 1.9194896594854072e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.320828378200531,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 378,
|
|
"train_speed(iter/s)": 0.008541
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 642.0,
|
|
"completions/mean_length": 369.734375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.21977384749202666,
|
|
"grad_norm": 0.4676888604112763,
|
|
"kl": 0.01898193359375,
|
|
"learning_rate": 4.925326223824873e-07,
|
|
"loss": 1.8967632058775052e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2734375,
|
|
"reward_std": 0.3224472105503082,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 379,
|
|
"train_speed(iter/s)": 0.008557
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1185.0,
|
|
"completions/mean_length": 397.703125,
|
|
"completions/min_length": 129.0,
|
|
"epoch": 0.2203537257175993,
|
|
"grad_norm": 0.48332346717530006,
|
|
"kl": 0.0159912109375,
|
|
"learning_rate": 4.924785206046699e-07,
|
|
"loss": 1.6042933566495776e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.3336407244205475,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 380,
|
|
"train_speed(iter/s)": 0.008569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 904.0,
|
|
"completions/mean_length": 358.65625,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.22093360394317194,
|
|
"grad_norm": 0.5416526347933004,
|
|
"kl": 0.0247802734375,
|
|
"learning_rate": 4.924242268788725e-07,
|
|
"loss": 2.4784905690466985e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59765625,
|
|
"reward_std": 0.22884789109230042,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 381,
|
|
"train_speed(iter/s)": 0.008583
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2435.0,
|
|
"completions/mean_length": 414.1484375,
|
|
"completions/min_length": 127.0,
|
|
"epoch": 0.22151348216874456,
|
|
"grad_norm": 0.5234061165602867,
|
|
"kl": 0.01922607421875,
|
|
"learning_rate": 4.923697412530154e-07,
|
|
"loss": 1.925699871208053e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.3819429278373718,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 382,
|
|
"train_speed(iter/s)": 0.008591
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1623.0,
|
|
"completions/mean_length": 358.234375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.22209336039431718,
|
|
"grad_norm": 0.5662616396045138,
|
|
"kl": 0.01922607421875,
|
|
"learning_rate": 4.923150637751875e-07,
|
|
"loss": 1.9217839508200996e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.38846004009246826,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 383,
|
|
"train_speed(iter/s)": 0.008603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1132.0,
|
|
"completions/mean_length": 388.34375,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.22267323861988983,
|
|
"grad_norm": 0.5158717513838648,
|
|
"kl": 0.09332275390625,
|
|
"learning_rate": 4.922601944936479e-07,
|
|
"loss": 9.339833195554093e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.4089977443218231,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 384,
|
|
"train_speed(iter/s)": 0.008616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 549.0,
|
|
"completions/mean_length": 301.484375,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.22325311684546245,
|
|
"grad_norm": 0.5387987764900297,
|
|
"kl": 0.25726318359375,
|
|
"learning_rate": 4.922051334568244e-07,
|
|
"loss": 0.00025730434572324157,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.2975226640701294,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 385,
|
|
"train_speed(iter/s)": 0.008568
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1435.0,
|
|
"completions/mean_length": 434.9453125,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.22383299507103507,
|
|
"grad_norm": 0.4778901050348148,
|
|
"kl": 0.016357421875,
|
|
"learning_rate": 4.921498807133142e-07,
|
|
"loss": 1.6353384125977755e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.3375301659107208,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 386,
|
|
"train_speed(iter/s)": 0.008581
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 747.0,
|
|
"completions/mean_length": 343.1875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.22441287329660772,
|
|
"grad_norm": 0.562037484463061,
|
|
"kl": 0.021728515625,
|
|
"learning_rate": 4.920944363118838e-07,
|
|
"loss": 2.1722378733102232e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.33213484287261963,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 387,
|
|
"train_speed(iter/s)": 0.008596
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1396.0,
|
|
"completions/mean_length": 401.3828125,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.22499275152218035,
|
|
"grad_norm": 0.48391981341281,
|
|
"kl": 0.0201416015625,
|
|
"learning_rate": 4.920388003014689e-07,
|
|
"loss": 2.0115550796617754e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.3190588653087616,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 388,
|
|
"train_speed(iter/s)": 0.008609
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1422.0,
|
|
"completions/mean_length": 353.3046875,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.22557262974775297,
|
|
"grad_norm": 0.5473218032194894,
|
|
"kl": 0.0208740234375,
|
|
"learning_rate": 4.919829727311741e-07,
|
|
"loss": 2.0841514924541116e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.35975906252861023,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 389,
|
|
"train_speed(iter/s)": 0.008622
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2631.0,
|
|
"completions/mean_length": 351.46875,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.2261525079733256,
|
|
"grad_norm": 0.5991607543960756,
|
|
"kl": 0.01885986328125,
|
|
"learning_rate": 4.919269536502731e-07,
|
|
"loss": 1.8878279661294073e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.35250747203826904,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 390,
|
|
"train_speed(iter/s)": 0.008626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1519.0,
|
|
"completions/mean_length": 370.7890625,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.22673238619889824,
|
|
"grad_norm": 0.5487522038588682,
|
|
"kl": 0.0218505859375,
|
|
"learning_rate": 4.918707431082087e-07,
|
|
"loss": 2.185619086958468e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3046875,
|
|
"reward_std": 0.34937870502471924,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 391,
|
|
"train_speed(iter/s)": 0.008638
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 958.0,
|
|
"completions/mean_length": 402.921875,
|
|
"completions/min_length": 150.0,
|
|
"epoch": 0.22731226442447086,
|
|
"grad_norm": 0.4761039661792473,
|
|
"kl": 0.01849365234375,
|
|
"learning_rate": 4.91814341154593e-07,
|
|
"loss": 1.8492377421353012e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.2981703579425812,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 392,
|
|
"train_speed(iter/s)": 0.008653
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 839.0,
|
|
"completions/mean_length": 364.84375,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.22789214265004348,
|
|
"grad_norm": 0.6398989659809242,
|
|
"kl": 0.504150390625,
|
|
"learning_rate": 4.917577478392064e-07,
|
|
"loss": 0.0005048379534855485,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2578125,
|
|
"reward_std": 0.3179285526275635,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 393,
|
|
"train_speed(iter/s)": 0.008604
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 915.0,
|
|
"completions/mean_length": 327.1796875,
|
|
"completions/min_length": 123.0,
|
|
"epoch": 0.22847202087561613,
|
|
"grad_norm": 0.6001858353164203,
|
|
"kl": 0.02471923828125,
|
|
"learning_rate": 4.917009632119987e-07,
|
|
"loss": 2.4679626221768558e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.37823036313056946,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 394,
|
|
"train_speed(iter/s)": 0.008619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1116.0,
|
|
"completions/mean_length": 375.53125,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.22905189910118876,
|
|
"grad_norm": 0.452055802134404,
|
|
"kl": 0.0189208984375,
|
|
"learning_rate": 4.916439873230884e-07,
|
|
"loss": 1.889707891677972e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.21848636865615845,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 395,
|
|
"train_speed(iter/s)": 0.008633
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 820.0,
|
|
"completions/mean_length": 352.1484375,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.22963177732676138,
|
|
"grad_norm": 0.4594216853997808,
|
|
"kl": 0.0206298828125,
|
|
"learning_rate": 4.915868202227627e-07,
|
|
"loss": 2.0621164367184974e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.23083598911762238,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 396,
|
|
"train_speed(iter/s)": 0.008648
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 999.0,
|
|
"completions/mean_length": 368.6015625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.230211655552334,
|
|
"grad_norm": 0.5362927999873359,
|
|
"kl": 0.01953125,
|
|
"learning_rate": 4.915294619614777e-07,
|
|
"loss": 1.9580451407819055e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.31584328413009644,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 397,
|
|
"train_speed(iter/s)": 0.008662
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1531.0,
|
|
"completions/mean_length": 417.9140625,
|
|
"completions/min_length": 135.0,
|
|
"epoch": 0.23079153377790665,
|
|
"grad_norm": 0.4973465399868976,
|
|
"kl": 0.02471923828125,
|
|
"learning_rate": 4.914719125898582e-07,
|
|
"loss": 2.466671503498219e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.23828125,
|
|
"reward_std": 0.310037761926651,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 398,
|
|
"train_speed(iter/s)": 0.008674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1074.0,
|
|
"completions/mean_length": 359.46875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.23137141200347927,
|
|
"grad_norm": 0.5679817121316464,
|
|
"kl": 0.0406494140625,
|
|
"learning_rate": 4.914141721586977e-07,
|
|
"loss": 4.0706509025767446e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.3592149317264557,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 399,
|
|
"train_speed(iter/s)": 0.008625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1081.0,
|
|
"completions/mean_length": 377.640625,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.2319512902290519,
|
|
"grad_norm": 0.40353596578202977,
|
|
"kl": 0.01947021484375,
|
|
"learning_rate": 4.913562407189581e-07,
|
|
"loss": 1.945688018167857e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.28737977147102356,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 400,
|
|
"train_speed(iter/s)": 0.00864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1086.0,
|
|
"completions/mean_length": 368.625,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.23253116845462452,
|
|
"grad_norm": 0.4962804750183632,
|
|
"kl": 0.019775390625,
|
|
"learning_rate": 4.912981183217705e-07,
|
|
"loss": 1.9732320652110502e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.29296875,
|
|
"reward_std": 0.2878088653087616,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 401,
|
|
"train_speed(iter/s)": 0.008648
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1143.0,
|
|
"completions/mean_length": 405.2578125,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.23311104668019716,
|
|
"grad_norm": 0.46932142905532337,
|
|
"kl": 0.017333984375,
|
|
"learning_rate": 4.912398050184335e-07,
|
|
"loss": 1.7352031136397272e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.24283519387245178,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 402,
|
|
"train_speed(iter/s)": 0.008661
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1094.0,
|
|
"completions/mean_length": 353.25,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.2336909249057698,
|
|
"grad_norm": 0.46220477263266857,
|
|
"kl": 0.02410888671875,
|
|
"learning_rate": 4.911813008604153e-07,
|
|
"loss": 2.4106973796733655e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.22763928771018982,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 403,
|
|
"train_speed(iter/s)": 0.008673
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 849.0,
|
|
"completions/mean_length": 372.140625,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.2342708031313424,
|
|
"grad_norm": 0.46053836514627405,
|
|
"kl": 0.02130126953125,
|
|
"learning_rate": 4.911226058993517e-07,
|
|
"loss": 2.1306186681613326e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.28125,
|
|
"reward_std": 0.3113781809806824,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 404,
|
|
"train_speed(iter/s)": 0.008688
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1726.0,
|
|
"completions/mean_length": 413.5390625,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.23485068135691506,
|
|
"grad_norm": 0.6008528558116725,
|
|
"kl": 0.01953125,
|
|
"learning_rate": 4.910637201870476e-07,
|
|
"loss": 1.9547605916159227e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.30923938751220703,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 405,
|
|
"train_speed(iter/s)": 0.008695
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 817.0,
|
|
"completions/mean_length": 392.6015625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.23543055958248768,
|
|
"grad_norm": 0.46781832139235086,
|
|
"kl": 0.01959228515625,
|
|
"learning_rate": 4.910046437754758e-07,
|
|
"loss": 1.96003275050316e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2109375,
|
|
"reward_std": 0.33269578218460083,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 406,
|
|
"train_speed(iter/s)": 0.008697
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1756.0,
|
|
"completions/mean_length": 409.1796875,
|
|
"completions/min_length": 149.0,
|
|
"epoch": 0.2360104378080603,
|
|
"grad_norm": 0.45850445078416474,
|
|
"kl": 0.03094482421875,
|
|
"learning_rate": 4.909453767167773e-07,
|
|
"loss": 3.097659646300599e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.2919955849647522,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 407,
|
|
"train_speed(iter/s)": 0.008707
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1020.0,
|
|
"completions/mean_length": 351.234375,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.23659031603363292,
|
|
"grad_norm": 0.5124253273419671,
|
|
"kl": 0.0228271484375,
|
|
"learning_rate": 4.908859190632619e-07,
|
|
"loss": 2.2789346985518932e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.36323416233062744,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 408,
|
|
"train_speed(iter/s)": 0.00872
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/mean_length": 388.90625,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.23717019425920557,
|
|
"grad_norm": 0.5762008234337191,
|
|
"kl": 0.02655029296875,
|
|
"learning_rate": 4.908262708674072e-07,
|
|
"loss": 2.6503521439735778e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.4036370813846588,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 409,
|
|
"train_speed(iter/s)": 0.008735
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1176.0,
|
|
"completions/mean_length": 416.8828125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.2377500724847782,
|
|
"grad_norm": 0.45630178138918587,
|
|
"kl": 0.01666259765625,
|
|
"learning_rate": 4.907664321818591e-07,
|
|
"loss": 1.6628049706923775e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.2576806843280792,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 410,
|
|
"train_speed(iter/s)": 0.008748
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 759.0,
|
|
"completions/mean_length": 359.3671875,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.23832995071035082,
|
|
"grad_norm": 0.5260354599423821,
|
|
"kl": 0.0220947265625,
|
|
"learning_rate": 4.907064030594316e-07,
|
|
"loss": 2.2105805328465067e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.4015023708343506,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 411,
|
|
"train_speed(iter/s)": 0.008763
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1276.0,
|
|
"completions/mean_length": 362.8671875,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.23890982893592347,
|
|
"grad_norm": 0.3875420428935182,
|
|
"kl": 0.03924560546875,
|
|
"learning_rate": 4.906461835531068e-07,
|
|
"loss": 3.9357313653454185e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.23247367143630981,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 412,
|
|
"train_speed(iter/s)": 0.008775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 767.0,
|
|
"completions/mean_length": 366.15625,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.2394897071614961,
|
|
"grad_norm": 0.5607979218813319,
|
|
"kl": 0.0216064453125,
|
|
"learning_rate": 4.905857737160349e-07,
|
|
"loss": 2.1611405827570707e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6484375,
|
|
"reward_std": 0.3249962031841278,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 413,
|
|
"train_speed(iter/s)": 0.008791
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1038.0,
|
|
"completions/mean_length": 342.171875,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.2400695853870687,
|
|
"grad_norm": 0.5669908109274124,
|
|
"kl": 0.0357666015625,
|
|
"learning_rate": 4.90525173601534e-07,
|
|
"loss": 3.569632099242881e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.42196178436279297,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 414,
|
|
"train_speed(iter/s)": 0.008805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 626.0,
|
|
"completions/mean_length": 362.53125,
|
|
"completions/min_length": 137.0,
|
|
"epoch": 0.24064946361264133,
|
|
"grad_norm": 0.3967382146778516,
|
|
"kl": 0.020263671875,
|
|
"learning_rate": 4.904643832630901e-07,
|
|
"loss": 2.0278723241062835e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.2686898708343506,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 415,
|
|
"train_speed(iter/s)": 0.00882
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 706.0,
|
|
"completions/mean_length": 348.40625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.24122934183821398,
|
|
"grad_norm": 0.5183280739345995,
|
|
"kl": 0.021484375,
|
|
"learning_rate": 4.904034027543571e-07,
|
|
"loss": 2.146402584912721e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.33425477147102356,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 416,
|
|
"train_speed(iter/s)": 0.008834
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 668.0,
|
|
"completions/mean_length": 357.3828125,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.2418092200637866,
|
|
"grad_norm": 0.5733037316516694,
|
|
"kl": 0.09002685546875,
|
|
"learning_rate": 4.90342232129157e-07,
|
|
"loss": 9.016609692480415e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.2659901976585388,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 417,
|
|
"train_speed(iter/s)": 0.008788
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 6835.0,
|
|
"completions/mean_length": 413.375,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.24238909828935923,
|
|
"grad_norm": 0.5447347141102508,
|
|
"kl": 0.0230712890625,
|
|
"learning_rate": 4.902808714414793e-07,
|
|
"loss": 2.3102784325601533e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.31640625,
|
|
"reward_std": 0.36187899112701416,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 418,
|
|
"train_speed(iter/s)": 0.00877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 946.0,
|
|
"completions/mean_length": 368.203125,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.24296897651493188,
|
|
"grad_norm": 0.5829873255761325,
|
|
"kl": 0.0191650390625,
|
|
"learning_rate": 4.902193207454813e-07,
|
|
"loss": 1.9112218069494702e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.3298494815826416,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 419,
|
|
"train_speed(iter/s)": 0.008769
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 805.0,
|
|
"completions/mean_length": 355.8984375,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.2435488547405045,
|
|
"grad_norm": 0.5750959611343572,
|
|
"kl": 0.02166748046875,
|
|
"learning_rate": 4.901575800954881e-07,
|
|
"loss": 2.166442573070526e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.31640625,
|
|
"reward_std": 0.3616940379142761,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 420,
|
|
"train_speed(iter/s)": 0.008784
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 778.0,
|
|
"completions/mean_length": 338.3359375,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.24412873296607712,
|
|
"grad_norm": 0.5153736282381937,
|
|
"kl": 0.021484375,
|
|
"learning_rate": 4.900956495459923e-07,
|
|
"loss": 2.1488132915692404e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.26933756470680237,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 421,
|
|
"train_speed(iter/s)": 0.008798
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 716.0,
|
|
"completions/mean_length": 353.390625,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.24470861119164974,
|
|
"grad_norm": 0.5240259420760081,
|
|
"kl": 0.02880859375,
|
|
"learning_rate": 4.900335291516545e-07,
|
|
"loss": 2.8786609618691728e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.31154364347457886,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 422,
|
|
"train_speed(iter/s)": 0.008812
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1073.0,
|
|
"completions/mean_length": 366.609375,
|
|
"completions/min_length": 136.0,
|
|
"epoch": 0.2452884894172224,
|
|
"grad_norm": 0.5079858910989432,
|
|
"kl": 0.01910400390625,
|
|
"learning_rate": 4.899712189673022e-07,
|
|
"loss": 1.916975452331826e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.23046875,
|
|
"reward_std": 0.2921273708343506,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 423,
|
|
"train_speed(iter/s)": 0.008824
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 599.0,
|
|
"completions/mean_length": 352.453125,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.24586836764279502,
|
|
"grad_norm": 0.5042331737703238,
|
|
"kl": 0.0211181640625,
|
|
"learning_rate": 4.899087190479311e-07,
|
|
"loss": 2.1084291802253574e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2571197748184204,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 424,
|
|
"train_speed(iter/s)": 0.00884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 684.0,
|
|
"completions/mean_length": 316.515625,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.24644824586836764,
|
|
"grad_norm": 0.4268944455313682,
|
|
"kl": 0.02606201171875,
|
|
"learning_rate": 4.898460294487038e-07,
|
|
"loss": 2.602357380965259e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.23502269387245178,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 425,
|
|
"train_speed(iter/s)": 0.008854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 990.0,
|
|
"completions/mean_length": 379.5234375,
|
|
"completions/min_length": 147.0,
|
|
"epoch": 0.24702812409394026,
|
|
"grad_norm": 0.4540709897186892,
|
|
"kl": 0.01959228515625,
|
|
"learning_rate": 4.897831502249507e-07,
|
|
"loss": 1.9618375517893583e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.353934645652771,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 426,
|
|
"train_speed(iter/s)": 0.008865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 836.0,
|
|
"completions/mean_length": 337.125,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.2476080023195129,
|
|
"grad_norm": 0.5900914634321148,
|
|
"kl": 0.02374267578125,
|
|
"learning_rate": 4.897200814321693e-07,
|
|
"loss": 2.3712798792985268e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.314785361289978,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 427,
|
|
"train_speed(iter/s)": 0.00888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 851.0,
|
|
"completions/mean_length": 365.6875,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.24818788054508553,
|
|
"grad_norm": 0.5145607043416552,
|
|
"kl": 0.02374267578125,
|
|
"learning_rate": 4.896568231260247e-07,
|
|
"loss": 2.3712455003987998e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.3525753617286682,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 428,
|
|
"train_speed(iter/s)": 0.008892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 815.0,
|
|
"completions/mean_length": 358.4296875,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.24876775877065815,
|
|
"grad_norm": 0.5375924980742913,
|
|
"kl": 0.02294921875,
|
|
"learning_rate": 4.89593375362349e-07,
|
|
"loss": 2.29538454732392e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.2751619815826416,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 429,
|
|
"train_speed(iter/s)": 0.008904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 990.0,
|
|
"completions/mean_length": 393.328125,
|
|
"completions/min_length": 139.0,
|
|
"epoch": 0.2493476369962308,
|
|
"grad_norm": 0.4455903219444502,
|
|
"kl": 0.02325439453125,
|
|
"learning_rate": 4.895297381971415e-07,
|
|
"loss": 2.3204218450700864e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.310037761926651,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 430,
|
|
"train_speed(iter/s)": 0.008916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1041.0,
|
|
"completions/mean_length": 356.2421875,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.24992751522180343,
|
|
"grad_norm": 0.5079596530650989,
|
|
"kl": 0.01959228515625,
|
|
"learning_rate": 4.894659116865691e-07,
|
|
"loss": 1.9621003957581706e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3203125,
|
|
"reward_std": 0.281765878200531,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 431,
|
|
"train_speed(iter/s)": 0.008929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1486.0,
|
|
"completions/mean_length": 388.671875,
|
|
"completions/min_length": 144.0,
|
|
"epoch": 0.25050739344737605,
|
|
"grad_norm": 0.43251956502105804,
|
|
"kl": 0.02056884765625,
|
|
"learning_rate": 4.894018958869652e-07,
|
|
"loss": 2.0592164219124243e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.24525238573551178,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 432,
|
|
"train_speed(iter/s)": 0.008941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1104.0,
|
|
"completions/mean_length": 376.5234375,
|
|
"completions/min_length": 64.0,
|
|
"epoch": 0.2510872716729487,
|
|
"grad_norm": 0.45894565384680164,
|
|
"kl": 0.06951904296875,
|
|
"learning_rate": 4.893376908548306e-07,
|
|
"loss": 6.958348967600614e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33984375,
|
|
"reward_std": 0.3302597105503082,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 433,
|
|
"train_speed(iter/s)": 0.008953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 920.0,
|
|
"completions/mean_length": 401.7734375,
|
|
"completions/min_length": 142.0,
|
|
"epoch": 0.2516671498985213,
|
|
"grad_norm": 0.47698419434986955,
|
|
"kl": 0.05535888671875,
|
|
"learning_rate": 4.892732966468332e-07,
|
|
"loss": 5.5201810027938336e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.203125,
|
|
"reward_std": 0.2957531809806824,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 434,
|
|
"train_speed(iter/s)": 0.008965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9561.0,
|
|
"completions/mean_length": 440.6953125,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.25224702812409394,
|
|
"grad_norm": 0.5485436492118713,
|
|
"kl": 0.0228271484375,
|
|
"learning_rate": 4.892087133198078e-07,
|
|
"loss": 2.2820368030807003e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.3302597105503082,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 435,
|
|
"train_speed(iter/s)": 0.008918
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/mean_length": 357.4375,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.2528269063496666,
|
|
"grad_norm": 0.47212715977891506,
|
|
"kl": 0.12939453125,
|
|
"learning_rate": 4.891439409307559e-07,
|
|
"loss": 0.00012879652786068618,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37109375,
|
|
"reward_std": 0.2884975075721741,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 436,
|
|
"train_speed(iter/s)": 0.008931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1733.0,
|
|
"completions/mean_length": 342.4296875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.2534067845752392,
|
|
"grad_norm": 0.48186014850969505,
|
|
"kl": 0.14520263671875,
|
|
"learning_rate": 4.890789795368461e-07,
|
|
"loss": 0.000145630314364098,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.2649322748184204,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 437,
|
|
"train_speed(iter/s)": 0.008941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 652.0,
|
|
"completions/mean_length": 344.9453125,
|
|
"completions/min_length": 127.0,
|
|
"epoch": 0.25398666280081184,
|
|
"grad_norm": 0.5481588883340683,
|
|
"kl": 0.0224609375,
|
|
"learning_rate": 4.890138291954139e-07,
|
|
"loss": 2.244827919639647e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.2957531809806824,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 438,
|
|
"train_speed(iter/s)": 0.008955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1251.0,
|
|
"completions/mean_length": 348.6171875,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.2545665410263845,
|
|
"grad_norm": 0.5339146545671073,
|
|
"kl": 0.02496337890625,
|
|
"learning_rate": 4.889484899639612e-07,
|
|
"loss": 2.4935605324571952e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.3443976938724518,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 439,
|
|
"train_speed(iter/s)": 0.008967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1084.0,
|
|
"completions/mean_length": 377.6484375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.2551464192519571,
|
|
"grad_norm": 0.7288622532318912,
|
|
"kl": 0.45709228515625,
|
|
"learning_rate": 4.88882961900157e-07,
|
|
"loss": 0.0004558621731121093,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2578125,
|
|
"reward_std": 0.3898337185382843,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.484375,
|
|
"rewards/FMTORM/std": 0.0873381495475769,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 440,
|
|
"train_speed(iter/s)": 0.008977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1278.0,
|
|
"completions/mean_length": 361.640625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.25572629747752973,
|
|
"grad_norm": 0.47584020229306767,
|
|
"kl": 0.0703125,
|
|
"learning_rate": 4.888172450618367e-07,
|
|
"loss": 7.041834032861516e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.26953125,
|
|
"reward_std": 0.26114001870155334,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 441,
|
|
"train_speed(iter/s)": 0.008989
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1292.0,
|
|
"completions/mean_length": 372.2578125,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.2563061757031023,
|
|
"grad_norm": 0.4677580053627263,
|
|
"kl": 0.02166748046875,
|
|
"learning_rate": 4.887513395070024e-07,
|
|
"loss": 2.169266554119531e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.23409250378608704,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 442,
|
|
"train_speed(iter/s)": 0.009001
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 757.0,
|
|
"completions/mean_length": 350.9453125,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.256886053928675,
|
|
"grad_norm": 0.5516049787298982,
|
|
"kl": 0.0416259765625,
|
|
"learning_rate": 4.886852452938228e-07,
|
|
"loss": 4.151882603764534e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.3184111714363098,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 443,
|
|
"train_speed(iter/s)": 0.009014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 807.0,
|
|
"completions/mean_length": 366.0390625,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.2574659321542476,
|
|
"grad_norm": 0.489599365891337,
|
|
"kl": 0.0213623046875,
|
|
"learning_rate": 4.886189624806333e-07,
|
|
"loss": 2.137709998351056e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.2781400680541992,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 444,
|
|
"train_speed(iter/s)": 0.009028
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1034.0,
|
|
"completions/mean_length": 369.578125,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.2580458103798202,
|
|
"grad_norm": 0.46089752816635016,
|
|
"kl": 0.02203369140625,
|
|
"learning_rate": 4.885524911259353e-07,
|
|
"loss": 2.2019998141331598e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.26635944843292236,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 445,
|
|
"train_speed(iter/s)": 0.009041
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1950.0,
|
|
"completions/mean_length": 388.6328125,
|
|
"completions/min_length": 136.0,
|
|
"epoch": 0.25862568860539287,
|
|
"grad_norm": 0.4932337795306393,
|
|
"kl": 0.02191162109375,
|
|
"learning_rate": 4.884858312883968e-07,
|
|
"loss": 2.191671956097707e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.3113781809806824,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 446,
|
|
"train_speed(iter/s)": 0.00905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1074.0,
|
|
"completions/mean_length": 362.96875,
|
|
"completions/min_length": 138.0,
|
|
"epoch": 0.2592055668309655,
|
|
"grad_norm": 0.5151475236964111,
|
|
"kl": 0.0604248046875,
|
|
"learning_rate": 4.884189830268526e-07,
|
|
"loss": 6.050239608157426e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.32846710085868835,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 447,
|
|
"train_speed(iter/s)": 0.009061
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1225.0,
|
|
"completions/mean_length": 338.6796875,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.2597854450565381,
|
|
"grad_norm": 0.5048988752401018,
|
|
"kl": 0.02593994140625,
|
|
"learning_rate": 4.883519464003029e-07,
|
|
"loss": 2.595646947156638e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.61328125,
|
|
"reward_std": 0.24434107542037964,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 448,
|
|
"train_speed(iter/s)": 0.009073
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 986.0,
|
|
"completions/mean_length": 381.625,
|
|
"completions/min_length": 137.0,
|
|
"epoch": 0.26036532328211076,
|
|
"grad_norm": 0.5615701664475433,
|
|
"kl": 0.0213623046875,
|
|
"learning_rate": 4.882847214679151e-07,
|
|
"loss": 2.1385894797276706e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.296875,
|
|
"reward_std": 0.40577587485313416,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 449,
|
|
"train_speed(iter/s)": 0.009085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1181.0,
|
|
"completions/mean_length": 361.859375,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.2609452015076834,
|
|
"grad_norm": 0.37262247434078904,
|
|
"kl": 0.04833984375,
|
|
"learning_rate": 4.882173082890221e-07,
|
|
"loss": 4.8364025133196265e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 450,
|
|
"train_speed(iter/s)": 0.009098
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 746.0,
|
|
"completions/mean_length": 337.7109375,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.261525079733256,
|
|
"grad_norm": 0.5367910763300202,
|
|
"kl": 0.02447509765625,
|
|
"learning_rate": 4.881497069231234e-07,
|
|
"loss": 2.446001599309966e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.3376619815826416,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 451,
|
|
"train_speed(iter/s)": 0.009112
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1010.0,
|
|
"completions/mean_length": 381.96875,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.26210495795882865,
|
|
"grad_norm": 0.46877839914567,
|
|
"kl": 0.0211181640625,
|
|
"learning_rate": 4.880819174298843e-07,
|
|
"loss": 2.1127420041011646e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.3336070775985718,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 452,
|
|
"train_speed(iter/s)": 0.009124
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 912.0,
|
|
"completions/mean_length": 351.4609375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.26268483618440125,
|
|
"grad_norm": 0.633954250130928,
|
|
"kl": 0.02252197265625,
|
|
"learning_rate": 4.880139398691363e-07,
|
|
"loss": 2.2540250938618556e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.3272815942764282,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 453,
|
|
"train_speed(iter/s)": 0.009136
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 652.0,
|
|
"completions/mean_length": 301.796875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.2632647144099739,
|
|
"grad_norm": 0.45802005674179846,
|
|
"kl": 0.04241943359375,
|
|
"learning_rate": 4.879457743008767e-07,
|
|
"loss": 4.2389510781504214e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.23762214183807373,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 454,
|
|
"train_speed(iter/s)": 0.009151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1212.0,
|
|
"completions/mean_length": 406.0625,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.26384459263554655,
|
|
"grad_norm": 0.6066045333065305,
|
|
"kl": 0.01806640625,
|
|
"learning_rate": 4.878774207852693e-07,
|
|
"loss": 1.8055065083899535e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.44500380754470825,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 455,
|
|
"train_speed(iter/s)": 0.009163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 977.0,
|
|
"completions/mean_length": 342.3828125,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.26442447086111914,
|
|
"grad_norm": 0.38737034238244145,
|
|
"kl": 0.026611328125,
|
|
"learning_rate": 4.878088793826427e-07,
|
|
"loss": 2.6647172489902005e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.278787761926651,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 456,
|
|
"train_speed(iter/s)": 0.009177
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1388.0,
|
|
"completions/mean_length": 349.140625,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.2650043490866918,
|
|
"grad_norm": 0.5874000703039106,
|
|
"kl": 0.0262451171875,
|
|
"learning_rate": 4.877401501534926e-07,
|
|
"loss": 2.6295871066395193e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.3733959496021271,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 457,
|
|
"train_speed(iter/s)": 0.009186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1307.0,
|
|
"completions/mean_length": 395.28125,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.26558422731226444,
|
|
"grad_norm": 0.3966125790654687,
|
|
"kl": 0.06195068359375,
|
|
"learning_rate": 4.876712331584797e-07,
|
|
"loss": 6.191872671479359e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.29000747203826904,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 458,
|
|
"train_speed(iter/s)": 0.009198
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 906.0,
|
|
"completions/mean_length": 362.875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.26616410553783704,
|
|
"grad_norm": 0.5037034364378293,
|
|
"kl": 0.02294921875,
|
|
"learning_rate": 4.876021284584304e-07,
|
|
"loss": 2.294294063176494e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.29296875,
|
|
"reward_std": 0.34990188479423523,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 459,
|
|
"train_speed(iter/s)": 0.009199
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 799.0,
|
|
"completions/mean_length": 337.046875,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.2667439837634097,
|
|
"grad_norm": 0.5489412736690018,
|
|
"kl": 0.0238037109375,
|
|
"learning_rate": 4.87532836114337e-07,
|
|
"loss": 2.374410178163089e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3046875,
|
|
"reward_std": 0.3869541585445404,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 460,
|
|
"train_speed(iter/s)": 0.009212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1765.0,
|
|
"completions/mean_length": 374.3203125,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.26732386198898234,
|
|
"grad_norm": 0.6375546888456002,
|
|
"kl": 0.0302734375,
|
|
"learning_rate": 4.874633561873577e-07,
|
|
"loss": 3.033534267160576e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.140625,
|
|
"reward_std": 0.37780123949050903,
|
|
"rewards/CSTORM/mean": 0.19140625,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 461,
|
|
"train_speed(iter/s)": 0.009221
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 825.0,
|
|
"completions/mean_length": 371.5859375,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.26790374021455493,
|
|
"grad_norm": 0.4971924086845082,
|
|
"kl": 0.163330078125,
|
|
"learning_rate": 4.873936887388157e-07,
|
|
"loss": 0.0001631513296160847,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.257100909948349,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 462,
|
|
"train_speed(iter/s)": 0.009234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 597.0,
|
|
"completions/mean_length": 361.4921875,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.2684836184401276,
|
|
"grad_norm": 0.4805749249128367,
|
|
"kl": 0.023193359375,
|
|
"learning_rate": 4.873238338301999e-07,
|
|
"loss": 2.323469379916787e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.2728765904903412,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 463,
|
|
"train_speed(iter/s)": 0.009248
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 959.0,
|
|
"completions/mean_length": 367.015625,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.26906349666570023,
|
|
"grad_norm": 0.524945899572977,
|
|
"kl": 0.0272216796875,
|
|
"learning_rate": 4.872537915231648e-07,
|
|
"loss": 2.7272737497696653e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.28631776571273804,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 464,
|
|
"train_speed(iter/s)": 0.009261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1432.0,
|
|
"completions/mean_length": 370.3828125,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.2696433748912728,
|
|
"grad_norm": 0.5168350166068164,
|
|
"kl": 0.0260009765625,
|
|
"learning_rate": 4.871835618795302e-07,
|
|
"loss": 2.6021334633696824e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.2517244815826416,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 465,
|
|
"train_speed(iter/s)": 0.009272
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 934.0,
|
|
"completions/mean_length": 366.6875,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.2702232531168455,
|
|
"grad_norm": 0.4255438403331787,
|
|
"kl": 0.0235595703125,
|
|
"learning_rate": 4.871131449612812e-07,
|
|
"loss": 2.353258605580777e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.29221415519714355,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 466,
|
|
"train_speed(iter/s)": 0.009286
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1655.0,
|
|
"completions/mean_length": 366.15625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.27080313134241807,
|
|
"grad_norm": 0.5487129480072858,
|
|
"kl": 0.02508544921875,
|
|
"learning_rate": 4.870425408305683e-07,
|
|
"loss": 2.509447767806705e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.3575604557991028,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 467,
|
|
"train_speed(iter/s)": 0.009295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 650.0,
|
|
"completions/mean_length": 340.890625,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.2713830095679907,
|
|
"grad_norm": 0.4780791322498501,
|
|
"kl": 0.02593994140625,
|
|
"learning_rate": 4.869717495497072e-07,
|
|
"loss": 2.5956735044019297e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.3532869815826416,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 468,
|
|
"train_speed(iter/s)": 0.00931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 767.0,
|
|
"completions/mean_length": 366.03125,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.27196288779356337,
|
|
"grad_norm": 0.43279311050086106,
|
|
"kl": 0.02392578125,
|
|
"learning_rate": 4.869007711811785e-07,
|
|
"loss": 2.3876100385678e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.25577935576438904,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 469,
|
|
"train_speed(iter/s)": 0.009324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1699.0,
|
|
"completions/mean_length": 418.1875,
|
|
"completions/min_length": 128.0,
|
|
"epoch": 0.27254276601913596,
|
|
"grad_norm": 0.4041167592227248,
|
|
"kl": 0.021484375,
|
|
"learning_rate": 4.868296057876286e-07,
|
|
"loss": 2.147430132026784e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.2529551684856415,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 470,
|
|
"train_speed(iter/s)": 0.009333
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1046.0,
|
|
"completions/mean_length": 350.15625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.2731226442447086,
|
|
"grad_norm": 0.5331626308226958,
|
|
"kl": 0.0252685546875,
|
|
"learning_rate": 4.867582534318681e-07,
|
|
"loss": 2.5292241843999363e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30078125,
|
|
"reward_std": 0.3142244815826416,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 471,
|
|
"train_speed(iter/s)": 0.009345
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 710.0,
|
|
"completions/mean_length": 357.21875,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.27370252247028126,
|
|
"grad_norm": 0.3513449023657793,
|
|
"kl": 0.0225830078125,
|
|
"learning_rate": 4.866867141768734e-07,
|
|
"loss": 2.259055690956302e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.1755007952451706,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 472,
|
|
"train_speed(iter/s)": 0.009359
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 395.5859375,
|
|
"completions/min_length": 134.0,
|
|
"epoch": 0.27428240069585386,
|
|
"grad_norm": 0.39547931222307336,
|
|
"kl": 0.0205078125,
|
|
"learning_rate": 4.866149880857855e-07,
|
|
"loss": 2.050572948064655e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.15625,
|
|
"reward_std": 0.22664928436279297,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.453125,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 473,
|
|
"train_speed(iter/s)": 0.009294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 796.0,
|
|
"completions/mean_length": 334.2890625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.2748622789214265,
|
|
"grad_norm": 0.5452667680349312,
|
|
"kl": 0.02423095703125,
|
|
"learning_rate": 4.8654307522191e-07,
|
|
"loss": 2.426551509415731e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37109375,
|
|
"reward_std": 0.356912761926651,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 474,
|
|
"train_speed(iter/s)": 0.009308
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1008.0,
|
|
"completions/mean_length": 359.8984375,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.27544215714699916,
|
|
"grad_norm": 0.4940492088506321,
|
|
"kl": 0.0350341796875,
|
|
"learning_rate": 4.864709756487181e-07,
|
|
"loss": 3.499436206766404e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.3213253915309906,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 475,
|
|
"train_speed(iter/s)": 0.009321
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 630.0,
|
|
"completions/mean_length": 335.203125,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.27602203537257175,
|
|
"grad_norm": 0.5072653259921915,
|
|
"kl": 0.02825927734375,
|
|
"learning_rate": 4.86398689429845e-07,
|
|
"loss": 2.8284697691560723e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.30598288774490356,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 476,
|
|
"train_speed(iter/s)": 0.009334
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 681.0,
|
|
"completions/mean_length": 343.1015625,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.2766019135981444,
|
|
"grad_norm": 0.5396745997304482,
|
|
"kl": 0.025146484375,
|
|
"learning_rate": 4.863262166290912e-07,
|
|
"loss": 2.5103036023210734e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3203125,
|
|
"reward_std": 0.44491297006607056,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 477,
|
|
"train_speed(iter/s)": 0.009347
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 888.0,
|
|
"completions/mean_length": 338.4921875,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.277181791823717,
|
|
"grad_norm": 0.5032582060141829,
|
|
"kl": 0.030517578125,
|
|
"learning_rate": 4.862535573104217e-07,
|
|
"loss": 3.055471461266279e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.35975906252861023,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 478,
|
|
"train_speed(iter/s)": 0.00936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1575.0,
|
|
"completions/mean_length": 338.453125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.27776167004928964,
|
|
"grad_norm": 0.5006685348343924,
|
|
"kl": 0.02777099609375,
|
|
"learning_rate": 4.861807115379658e-07,
|
|
"loss": 2.772811239992734e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.2775791883468628,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 479,
|
|
"train_speed(iter/s)": 0.00937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2505.0,
|
|
"completions/mean_length": 341.6796875,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.2783415482748623,
|
|
"grad_norm": 0.5495460190970202,
|
|
"kl": 0.0267333984375,
|
|
"learning_rate": 4.86107679376018e-07,
|
|
"loss": 2.6742291083792225e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.26454412937164307,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 480,
|
|
"train_speed(iter/s)": 0.009375
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1158.0,
|
|
"completions/mean_length": 374.0859375,
|
|
"completions/min_length": 123.0,
|
|
"epoch": 0.2789214265004349,
|
|
"grad_norm": 0.4858879983802429,
|
|
"kl": 0.0260009765625,
|
|
"learning_rate": 4.860344608890368e-07,
|
|
"loss": 2.6020570658147335e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.2885015904903412,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 481,
|
|
"train_speed(iter/s)": 0.009384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 858.0,
|
|
"completions/mean_length": 331.6796875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.27950130472600754,
|
|
"grad_norm": 0.6424815007926947,
|
|
"kl": 0.0252685546875,
|
|
"learning_rate": 4.859610561416455e-07,
|
|
"loss": 2.5241974071832374e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.57421875,
|
|
"reward_std": 0.33928078413009644,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 482,
|
|
"train_speed(iter/s)": 0.009397
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/mean_length": 355.953125,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.2800811829515802,
|
|
"grad_norm": 0.520958443182139,
|
|
"kl": 0.02374267578125,
|
|
"learning_rate": 4.858874651986314e-07,
|
|
"loss": 2.376912561885547e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3671875,
|
|
"reward_std": 0.36446163058280945,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 483,
|
|
"train_speed(iter/s)": 0.009408
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 976.0,
|
|
"completions/mean_length": 370.8984375,
|
|
"completions/min_length": 133.0,
|
|
"epoch": 0.2806610611771528,
|
|
"grad_norm": 0.5225006972225628,
|
|
"kl": 0.02392578125,
|
|
"learning_rate": 4.858136881249464e-07,
|
|
"loss": 2.3908047296572477e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.3831776976585388,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 484,
|
|
"train_speed(iter/s)": 0.00942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1768.0,
|
|
"completions/mean_length": 395.734375,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.28124093940272543,
|
|
"grad_norm": 0.5488221448670664,
|
|
"kl": 0.020263671875,
|
|
"learning_rate": 4.857397249857066e-07,
|
|
"loss": 2.027053778874688e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.265625,
|
|
"reward_std": 0.3484525680541992,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 485,
|
|
"train_speed(iter/s)": 0.009413
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 936.0,
|
|
"completions/mean_length": 318.4921875,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.2818208176282981,
|
|
"grad_norm": 0.48021468530366096,
|
|
"kl": 0.02996826171875,
|
|
"learning_rate": 4.856655758461925e-07,
|
|
"loss": 2.988072628795635e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.25577935576438904,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 486,
|
|
"train_speed(iter/s)": 0.009425
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1078.0,
|
|
"completions/mean_length": 351.109375,
|
|
"completions/min_length": 123.0,
|
|
"epoch": 0.2824006958538707,
|
|
"grad_norm": 0.5634920168092982,
|
|
"kl": 0.026611328125,
|
|
"learning_rate": 4.855912407718485e-07,
|
|
"loss": 2.658888479345478e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.25390625,
|
|
"reward_std": 0.4530799388885498,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.515625,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 487,
|
|
"train_speed(iter/s)": 0.009438
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 793.0,
|
|
"completions/mean_length": 371.15625,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.2829805740794433,
|
|
"grad_norm": 0.4088970846578649,
|
|
"kl": 0.04022216796875,
|
|
"learning_rate": 4.855167198282834e-07,
|
|
"loss": 4.010339034721255e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.26171875,
|
|
"reward_std": 0.34383678436279297,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 488,
|
|
"train_speed(iter/s)": 0.009451
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 584.0,
|
|
"completions/mean_length": 336.03125,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.2835604523050159,
|
|
"grad_norm": 0.5385394373245845,
|
|
"kl": 0.02874755859375,
|
|
"learning_rate": 4.854420130812696e-07,
|
|
"loss": 2.871574724849779e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.330410361289978,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 489,
|
|
"train_speed(iter/s)": 0.009464
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1184.0,
|
|
"completions/mean_length": 358.6875,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.28414033053058857,
|
|
"grad_norm": 0.5514045034332898,
|
|
"kl": 0.02667236328125,
|
|
"learning_rate": 4.853671205967439e-07,
|
|
"loss": 2.6700774469645694e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.311788409948349,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 490,
|
|
"train_speed(iter/s)": 0.009476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 836.0,
|
|
"completions/mean_length": 314.921875,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.2847202087561612,
|
|
"grad_norm": 0.5393834673780427,
|
|
"kl": 0.03125,
|
|
"learning_rate": 4.85292042440807e-07,
|
|
"loss": 3.1203166145132855e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.2860393524169922,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 491,
|
|
"train_speed(iter/s)": 0.009487
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1708.0,
|
|
"completions/mean_length": 346.953125,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.2853000869817338,
|
|
"grad_norm": 0.45094282418332354,
|
|
"kl": 0.0260009765625,
|
|
"learning_rate": 4.852167786797233e-07,
|
|
"loss": 2.6006520783994347e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.2847439646720886,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 492,
|
|
"train_speed(iter/s)": 0.009496
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 664.0,
|
|
"completions/mean_length": 354.453125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.28587996520730646,
|
|
"grad_norm": 0.44727662790867506,
|
|
"kl": 0.0350341796875,
|
|
"learning_rate": 4.85141329379921e-07,
|
|
"loss": 3.5089375160168856e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.2416265904903412,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 493,
|
|
"train_speed(iter/s)": 0.009509
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1040.0,
|
|
"completions/mean_length": 311.421875,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.2864598434328791,
|
|
"grad_norm": 0.5316382608660587,
|
|
"kl": 0.032470703125,
|
|
"learning_rate": 4.850656946079923e-07,
|
|
"loss": 3.249772635172121e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3515625,
|
|
"reward_std": 0.32039928436279297,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 494,
|
|
"train_speed(iter/s)": 0.00952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 572.0,
|
|
"completions/mean_length": 327.609375,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.2870397216584517,
|
|
"grad_norm": 0.41270897690670433,
|
|
"kl": 0.028564453125,
|
|
"learning_rate": 4.849898744306929e-07,
|
|
"loss": 2.8565837055793963e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.2692508101463318,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 495,
|
|
"train_speed(iter/s)": 0.009535
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1358.0,
|
|
"completions/mean_length": 351.203125,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.28761959988402436,
|
|
"grad_norm": 0.49576608830532837,
|
|
"kl": 0.032958984375,
|
|
"learning_rate": 4.849138689149421e-07,
|
|
"loss": 3.302330878796056e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2614383101463318,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 496,
|
|
"train_speed(iter/s)": 0.009545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1054.0,
|
|
"completions/mean_length": 330.2578125,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.288199478109597,
|
|
"grad_norm": 0.5598383557607869,
|
|
"kl": 0.02838134765625,
|
|
"learning_rate": 4.848376781278231e-07,
|
|
"loss": 2.833348116837442e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.299247145652771,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 497,
|
|
"train_speed(iter/s)": 0.009557
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1450.0,
|
|
"completions/mean_length": 338.6640625,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.2887793563351696,
|
|
"grad_norm": 0.5540979181568668,
|
|
"kl": 0.02978515625,
|
|
"learning_rate": 4.84761302136582e-07,
|
|
"loss": 2.9738344892393798e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.3489684462547302,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 498,
|
|
"train_speed(iter/s)": 0.009567
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 619.0,
|
|
"completions/mean_length": 334.2109375,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.28935923456074225,
|
|
"grad_norm": 0.43756131559932954,
|
|
"kl": 0.03045654296875,
|
|
"learning_rate": 4.846847410086292e-07,
|
|
"loss": 3.0436665838351473e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.19759789109230042,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 499,
|
|
"train_speed(iter/s)": 0.009581
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 591.0,
|
|
"completions/mean_length": 337.4140625,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.2899391127863149,
|
|
"grad_norm": 0.5180047515891838,
|
|
"kl": 0.0697021484375,
|
|
"learning_rate": 4.846079948115378e-07,
|
|
"loss": 6.966946239117533e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.3262236714363098,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 500,
|
|
"train_speed(iter/s)": 0.009594
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 718.0,
|
|
"completions/mean_length": 321.9921875,
|
|
"completions/min_length": 148.0,
|
|
"epoch": 0.2905189910118875,
|
|
"grad_norm": 0.5709049216332543,
|
|
"kl": 0.12713623046875,
|
|
"learning_rate": 4.845310636130444e-07,
|
|
"loss": 0.00012690876610577106,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.3284713327884674,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 501,
|
|
"train_speed(iter/s)": 0.009602
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 900.0,
|
|
"completions/mean_length": 340.7265625,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.29109886923746014,
|
|
"grad_norm": 0.3117307550937708,
|
|
"kl": 0.0333251953125,
|
|
"learning_rate": 4.844539474810494e-07,
|
|
"loss": 3.327765080030076e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.125,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 502,
|
|
"train_speed(iter/s)": 0.009614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1055.0,
|
|
"completions/mean_length": 323.890625,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.29167874746303274,
|
|
"grad_norm": 0.6233963386898691,
|
|
"kl": 0.104248046875,
|
|
"learning_rate": 4.843766464836155e-07,
|
|
"loss": 0.00010443136852700263,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.38299643993377686,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 503,
|
|
"train_speed(iter/s)": 0.009626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 816.0,
|
|
"completions/mean_length": 336.6328125,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.2922586256886054,
|
|
"grad_norm": 0.6399297350918488,
|
|
"kl": 0.02728271484375,
|
|
"learning_rate": 4.842991606889695e-07,
|
|
"loss": 2.7289714125799946e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.3737275004386902,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 504,
|
|
"train_speed(iter/s)": 0.009639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 721.0,
|
|
"completions/mean_length": 353.6015625,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.29283850391417804,
|
|
"grad_norm": 0.4029721370266875,
|
|
"kl": 0.0245361328125,
|
|
"learning_rate": 4.842214901655006e-07,
|
|
"loss": 2.4553393814130686e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.21913406252861023,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 505,
|
|
"train_speed(iter/s)": 0.009651
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 782.0,
|
|
"completions/mean_length": 328.2421875,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.29341838213975063,
|
|
"grad_norm": 0.5067197749642119,
|
|
"kl": 0.04443359375,
|
|
"learning_rate": 4.841436349817613e-07,
|
|
"loss": 4.422634083312005e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.32774800062179565,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 506,
|
|
"train_speed(iter/s)": 0.009602
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 657.0,
|
|
"completions/mean_length": 312.796875,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.2939982603653233,
|
|
"grad_norm": 0.46071690966706935,
|
|
"kl": 0.0360107421875,
|
|
"learning_rate": 4.840655952064674e-07,
|
|
"loss": 3.5912395105697215e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.73046875,
|
|
"reward_std": 0.23180712759494781,
|
|
"rewards/CSTORM/mean": 0.39453125,
|
|
"rewards/CSTORM/std": 0.20478858053684235,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8359375,
|
|
"rewards/VQAORM/std": 0.371787428855896,
|
|
"step": 507,
|
|
"train_speed(iter/s)": 0.009615
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1539.0,
|
|
"completions/mean_length": 368.203125,
|
|
"completions/min_length": 60.0,
|
|
"epoch": 0.29457813859089593,
|
|
"grad_norm": 0.5404247635408104,
|
|
"kl": 0.46673583984375,
|
|
"learning_rate": 4.839873709084971e-07,
|
|
"loss": 0.00046606952673755586,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.2426845133304596,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 508,
|
|
"train_speed(iter/s)": 0.009624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 760.0,
|
|
"completions/mean_length": 329.375,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.2951580168164685,
|
|
"grad_norm": 0.5502274680508231,
|
|
"kl": 0.028076171875,
|
|
"learning_rate": 4.839089621568916e-07,
|
|
"loss": 2.8074604415451176e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.2897101640701294,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 509,
|
|
"train_speed(iter/s)": 0.009638
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 959.0,
|
|
"completions/mean_length": 347.4375,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.2957378950420412,
|
|
"grad_norm": 0.44839411506432925,
|
|
"kl": 0.081298828125,
|
|
"learning_rate": 4.838303690208552e-07,
|
|
"loss": 8.159131539287046e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.32367467880249023,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 510,
|
|
"train_speed(iter/s)": 0.009649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 881.0,
|
|
"completions/mean_length": 332.046875,
|
|
"completions/min_length": 55.0,
|
|
"epoch": 0.2963177732676138,
|
|
"grad_norm": 0.47455213170988986,
|
|
"kl": 0.02691650390625,
|
|
"learning_rate": 4.837515915697546e-07,
|
|
"loss": 2.6935304049402475e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.21861818432807922,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 511,
|
|
"train_speed(iter/s)": 0.009662
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1028.0,
|
|
"completions/mean_length": 338.359375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.2968976514931864,
|
|
"grad_norm": 0.5087142444481082,
|
|
"kl": 0.02587890625,
|
|
"learning_rate": 4.836726298731193e-07,
|
|
"loss": 2.594956458779052e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.23828125,
|
|
"reward_std": 0.3298494815826416,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 512,
|
|
"train_speed(iter/s)": 0.009674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 638.0,
|
|
"completions/mean_length": 325.8984375,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.29747752971875907,
|
|
"grad_norm": 0.5245771581299422,
|
|
"kl": 0.02789306640625,
|
|
"learning_rate": 4.835934840006414e-07,
|
|
"loss": 2.7888094336958602e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.211453378200531,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 513,
|
|
"train_speed(iter/s)": 0.009688
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 580.0,
|
|
"completions/mean_length": 328.8828125,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.29805740794433166,
|
|
"grad_norm": 0.5291921290017736,
|
|
"kl": 0.030517578125,
|
|
"learning_rate": 4.835141540221756e-07,
|
|
"loss": 3.0521649023285136e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.3142244815826416,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 514,
|
|
"train_speed(iter/s)": 0.009701
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 836.0,
|
|
"completions/mean_length": 333.7421875,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.2986372861699043,
|
|
"grad_norm": 0.5032109821284366,
|
|
"kl": 0.106689453125,
|
|
"learning_rate": 4.834346400077391e-07,
|
|
"loss": 0.00010700618440750986,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.2336822748184204,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 515,
|
|
"train_speed(iter/s)": 0.009712
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1114.0,
|
|
"completions/mean_length": 351.6640625,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.29921716439547696,
|
|
"grad_norm": 0.48726019796620296,
|
|
"kl": 0.0283203125,
|
|
"learning_rate": 4.833549420275114e-07,
|
|
"loss": 2.8311660571489483e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.273953378200531,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 516,
|
|
"train_speed(iter/s)": 0.009723
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 662.0,
|
|
"completions/mean_length": 338.5625,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.29979704262104956,
|
|
"grad_norm": 0.41380672462813334,
|
|
"kl": 0.03240966796875,
|
|
"learning_rate": 4.832750601518345e-07,
|
|
"loss": 3.234517134842463e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.17416039109230042,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 517,
|
|
"train_speed(iter/s)": 0.009736
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1061.0,
|
|
"completions/mean_length": 341.7265625,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.3003769208466222,
|
|
"grad_norm": 0.49583544058167,
|
|
"kl": 0.0296630859375,
|
|
"learning_rate": 4.831949944512129e-07,
|
|
"loss": 2.960633355542086e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.2181890904903412,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 518,
|
|
"train_speed(iter/s)": 0.009748
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1288.0,
|
|
"completions/mean_length": 312.7890625,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.30095679907219486,
|
|
"grad_norm": 0.5301071434778198,
|
|
"kl": 0.0330810546875,
|
|
"learning_rate": 4.831147449963128e-07,
|
|
"loss": 3.299152012914419e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.3238932490348816,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 519,
|
|
"train_speed(iter/s)": 0.009758
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 909.0,
|
|
"completions/mean_length": 346.265625,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.30153667729776745,
|
|
"grad_norm": 0.5722743958845975,
|
|
"kl": 0.04461669921875,
|
|
"learning_rate": 4.830343118579629e-07,
|
|
"loss": 4.4531225285027176e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.4176432490348816,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 520,
|
|
"train_speed(iter/s)": 0.009706
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3680.0,
|
|
"completions/mean_length": 374.640625,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.3021165555233401,
|
|
"grad_norm": 0.4715808130950418,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 4.829536951071541e-07,
|
|
"loss": 3.2796386221889406e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.3233773708343506,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 521,
|
|
"train_speed(iter/s)": 0.009672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 573.0,
|
|
"completions/mean_length": 313.8984375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.30269643374891275,
|
|
"grad_norm": 0.5120201394841977,
|
|
"kl": 0.02899169921875,
|
|
"learning_rate": 4.828728948150396e-07,
|
|
"loss": 2.8987755285925232e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.258328378200531,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 522,
|
|
"train_speed(iter/s)": 0.009684
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/mean_length": 357.828125,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.30327631197448535,
|
|
"grad_norm": 0.5783345486956806,
|
|
"kl": 0.0279541015625,
|
|
"learning_rate": 4.827919110529337e-07,
|
|
"loss": 2.7976131605100818e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.3564305305480957,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 523,
|
|
"train_speed(iter/s)": 0.009697
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 757.0,
|
|
"completions/mean_length": 351.484375,
|
|
"completions/min_length": 127.0,
|
|
"epoch": 0.303856190200058,
|
|
"grad_norm": 0.5418800917114246,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 4.827107438923136e-07,
|
|
"loss": 3.2821531931404024e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.3364344835281372,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 524,
|
|
"train_speed(iter/s)": 0.009709
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 985.0,
|
|
"completions/mean_length": 341.5703125,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.3044360684256306,
|
|
"grad_norm": 0.5489066460555996,
|
|
"kl": 0.03143310546875,
|
|
"learning_rate": 4.826293934048177e-07,
|
|
"loss": 3.138860847684555e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59765625,
|
|
"reward_std": 0.3118072748184204,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 525,
|
|
"train_speed(iter/s)": 0.009721
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1032.0,
|
|
"completions/mean_length": 363.921875,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.30501594665120324,
|
|
"grad_norm": 0.4316115309570818,
|
|
"kl": 0.0240478515625,
|
|
"learning_rate": 4.825478596622468e-07,
|
|
"loss": 2.4090675651677884e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.27045938372612,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 526,
|
|
"train_speed(iter/s)": 0.00973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 972.0,
|
|
"completions/mean_length": 341.7265625,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.3055958248767759,
|
|
"grad_norm": 0.47023399125573356,
|
|
"kl": 0.02783203125,
|
|
"learning_rate": 4.82466142736563e-07,
|
|
"loss": 2.7847410819958895e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.24430333077907562,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 527,
|
|
"train_speed(iter/s)": 0.009741
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 794.0,
|
|
"completions/mean_length": 339.90625,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.3061757031023485,
|
|
"grad_norm": 0.5405314645848487,
|
|
"kl": 0.0322265625,
|
|
"learning_rate": 4.823842426998901e-07,
|
|
"loss": 3.219948848709464e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.3864383101463318,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 528,
|
|
"train_speed(iter/s)": 0.009753
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 721.0,
|
|
"completions/mean_length": 349.7890625,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.30675558132792113,
|
|
"grad_norm": 0.4978730246798589,
|
|
"kl": 0.0491943359375,
|
|
"learning_rate": 4.823021596245135e-07,
|
|
"loss": 4.917265323456377e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.269118994474411,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 529,
|
|
"train_speed(iter/s)": 0.009734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 669.0,
|
|
"completions/mean_length": 318.6875,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.3073354595534938,
|
|
"grad_norm": 0.5514166708994682,
|
|
"kl": 0.031005859375,
|
|
"learning_rate": 4.822198935828806e-07,
|
|
"loss": 3.099613240920007e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.28448036313056946,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 530,
|
|
"train_speed(iter/s)": 0.009747
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 960.0,
|
|
"completions/mean_length": 350.59375,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.3079153377790664,
|
|
"grad_norm": 0.560170341292171,
|
|
"kl": 0.03179931640625,
|
|
"learning_rate": 4.821374446475998e-07,
|
|
"loss": 3.171353091602214e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.3225978910923004,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 531,
|
|
"train_speed(iter/s)": 0.009758
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1019.0,
|
|
"completions/mean_length": 348.5625,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.30849521600463903,
|
|
"grad_norm": 0.5666914543656868,
|
|
"kl": 0.02984619140625,
|
|
"learning_rate": 4.82054812891441e-07,
|
|
"loss": 2.9840526622137986e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.3531551659107208,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 532,
|
|
"train_speed(iter/s)": 0.00977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1286.0,
|
|
"completions/mean_length": 336.234375,
|
|
"completions/min_length": 129.0,
|
|
"epoch": 0.3090750942302117,
|
|
"grad_norm": 0.38281711635463506,
|
|
"kl": 0.03460693359375,
|
|
"learning_rate": 4.819719983873356e-07,
|
|
"loss": 3.459706204012036e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.21969497203826904,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 533,
|
|
"train_speed(iter/s)": 0.009777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/mean_length": 308.140625,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.30965497245578427,
|
|
"grad_norm": 0.6591513972589031,
|
|
"kl": 0.03271484375,
|
|
"learning_rate": 4.818890012083763e-07,
|
|
"loss": 3.26778827002272e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.2933359742164612,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 534,
|
|
"train_speed(iter/s)": 0.009789
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1057.0,
|
|
"completions/mean_length": 352.3828125,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.3102348506813569,
|
|
"grad_norm": 0.5300862195262892,
|
|
"kl": 0.02630615234375,
|
|
"learning_rate": 4.818058214278167e-07,
|
|
"loss": 2.6301073376089334e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3359375,
|
|
"reward_std": 0.3064119815826416,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 535,
|
|
"train_speed(iter/s)": 0.009799
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 807.0,
|
|
"completions/mean_length": 347.1484375,
|
|
"completions/min_length": 127.0,
|
|
"epoch": 0.31081472890692957,
|
|
"grad_norm": 0.4678870041312948,
|
|
"kl": 0.02801513671875,
|
|
"learning_rate": 4.817224591190721e-07,
|
|
"loss": 2.802363633236382e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.24675828218460083,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 536,
|
|
"train_speed(iter/s)": 0.009811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1416.0,
|
|
"completions/mean_length": 368.40625,
|
|
"completions/min_length": 129.0,
|
|
"epoch": 0.31139460713250217,
|
|
"grad_norm": 0.4358624905849924,
|
|
"kl": 0.02984619140625,
|
|
"learning_rate": 4.816389143557185e-07,
|
|
"loss": 2.9819228075211868e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33984375,
|
|
"reward_std": 0.22330188751220703,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 537,
|
|
"train_speed(iter/s)": 0.009821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1037.0,
|
|
"completions/mean_length": 342.9375,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.3119744853580748,
|
|
"grad_norm": 0.5518166482624421,
|
|
"kl": 0.0302734375,
|
|
"learning_rate": 4.815551872114932e-07,
|
|
"loss": 3.0273757147369906e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.2650640904903412,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 538,
|
|
"train_speed(iter/s)": 0.009832
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 752.0,
|
|
"completions/mean_length": 307.6328125,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.3125543635836474,
|
|
"grad_norm": 0.5131355238847525,
|
|
"kl": 0.043212890625,
|
|
"learning_rate": 4.814712777602941e-07,
|
|
"loss": 4.326713315094821e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.31046685576438904,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 539,
|
|
"train_speed(iter/s)": 0.009845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1483.0,
|
|
"completions/mean_length": 336.703125,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.31313424180922006,
|
|
"grad_norm": 0.5635112710295342,
|
|
"kl": 0.0283203125,
|
|
"learning_rate": 4.813871860761804e-07,
|
|
"loss": 2.8267117158975452e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.2848758101463318,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 540,
|
|
"train_speed(iter/s)": 0.009854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1656.0,
|
|
"completions/mean_length": 394.1953125,
|
|
"completions/min_length": 146.0,
|
|
"epoch": 0.3137141200347927,
|
|
"grad_norm": 0.4221961249277467,
|
|
"kl": 0.02642822265625,
|
|
"learning_rate": 4.813029122333719e-07,
|
|
"loss": 2.6417414119350724e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.3645745813846588,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 541,
|
|
"train_speed(iter/s)": 0.009862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 856.0,
|
|
"completions/mean_length": 336.046875,
|
|
"completions/min_length": 139.0,
|
|
"epoch": 0.3142939982603653,
|
|
"grad_norm": 0.6006567714004777,
|
|
"kl": 0.033935546875,
|
|
"learning_rate": 4.812184563062493e-07,
|
|
"loss": 3.393915540073067e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.39920562505722046,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 542,
|
|
"train_speed(iter/s)": 0.009874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 931.0,
|
|
"completions/mean_length": 314.109375,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.31487387648593795,
|
|
"grad_norm": 0.5216177777003166,
|
|
"kl": 0.0457763671875,
|
|
"learning_rate": 4.81133818369354e-07,
|
|
"loss": 4.585651186062023e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.24525237083435059,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 543,
|
|
"train_speed(iter/s)": 0.009886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 919.0,
|
|
"completions/mean_length": 332.6875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.3154537547115106,
|
|
"grad_norm": 0.5034333085744941,
|
|
"kl": 0.0333251953125,
|
|
"learning_rate": 4.810489984973879e-07,
|
|
"loss": 3.3358628570567816e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.2354329228401184,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 544,
|
|
"train_speed(iter/s)": 0.009885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 693.0,
|
|
"completions/mean_length": 304.4296875,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.3160336329370832,
|
|
"grad_norm": 0.44315841417918034,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 4.809639967652137e-07,
|
|
"loss": 3.812746217590757e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.24358002841472626,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 545,
|
|
"train_speed(iter/s)": 0.009897
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 843.0,
|
|
"completions/mean_length": 338.2265625,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.31661351116265585,
|
|
"grad_norm": 0.6399379847356642,
|
|
"kl": 0.032958984375,
|
|
"learning_rate": 4.808788132478543e-07,
|
|
"loss": 3.297333751106635e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.422390878200531,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 546,
|
|
"train_speed(iter/s)": 0.009909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 857.0,
|
|
"completions/mean_length": 361.3671875,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.3171933893882285,
|
|
"grad_norm": 0.5219779689433417,
|
|
"kl": 0.0570068359375,
|
|
"learning_rate": 4.807934480204934e-07,
|
|
"loss": 5.687818702426739e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.2740851640701294,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 547,
|
|
"train_speed(iter/s)": 0.00992
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1143.0,
|
|
"completions/mean_length": 338.7421875,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.3177732676138011,
|
|
"grad_norm": 0.5785807490263075,
|
|
"kl": 0.0318603515625,
|
|
"learning_rate": 4.807079011584749e-07,
|
|
"loss": 3.182793443556875e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.3591981530189514,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 548,
|
|
"train_speed(iter/s)": 0.009931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 568.0,
|
|
"completions/mean_length": 313.1328125,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.31835314583937374,
|
|
"grad_norm": 0.5127938375326051,
|
|
"kl": 0.0322265625,
|
|
"learning_rate": 4.806221727373029e-07,
|
|
"loss": 3.224569445592351e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.30058759450912476,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 549,
|
|
"train_speed(iter/s)": 0.009944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 950.0,
|
|
"completions/mean_length": 341.984375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.31893302406494634,
|
|
"grad_norm": 0.46873442811345356,
|
|
"kl": 0.0791015625,
|
|
"learning_rate": 4.80536262832642e-07,
|
|
"loss": 7.915176684036851e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.33546337485313416,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 550,
|
|
"train_speed(iter/s)": 0.009954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1484.0,
|
|
"completions/mean_length": 397.2890625,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.319512902290519,
|
|
"grad_norm": 0.4145869496640962,
|
|
"kl": 0.02777099609375,
|
|
"learning_rate": 4.804501715203169e-07,
|
|
"loss": 2.7732741727959365e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2590211033821106,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 551,
|
|
"train_speed(iter/s)": 0.00995
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 994.0,
|
|
"completions/mean_length": 310.5234375,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.32009278051609164,
|
|
"grad_norm": 0.5292370859260453,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 4.803638988763121e-07,
|
|
"loss": 3.288583320681937e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.22962738573551178,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 552,
|
|
"train_speed(iter/s)": 0.00996
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 775.0,
|
|
"completions/mean_length": 348.703125,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.32067265874166423,
|
|
"grad_norm": 0.629246562694661,
|
|
"kl": 0.09820556640625,
|
|
"learning_rate": 4.802774449767728e-07,
|
|
"loss": 9.775245416676626e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.3388705849647522,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 553,
|
|
"train_speed(iter/s)": 0.009972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 652.0,
|
|
"completions/mean_length": 332.65625,
|
|
"completions/min_length": 128.0,
|
|
"epoch": 0.3212525369672369,
|
|
"grad_norm": 0.4669458583505929,
|
|
"kl": 0.033447265625,
|
|
"learning_rate": 4.801908098980034e-07,
|
|
"loss": 3.347734673297964e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2103765904903412,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 554,
|
|
"train_speed(iter/s)": 0.009983
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1156.0,
|
|
"completions/mean_length": 323.734375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.32183241519280953,
|
|
"grad_norm": 0.5222341382343001,
|
|
"kl": 0.03253173828125,
|
|
"learning_rate": 4.801039937164688e-07,
|
|
"loss": 3.25060827890411e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.3588140904903412,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 555,
|
|
"train_speed(iter/s)": 0.009994
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 811.0,
|
|
"completions/mean_length": 317.453125,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.3224122934183821,
|
|
"grad_norm": 0.5193402690562788,
|
|
"kl": 0.0316162109375,
|
|
"learning_rate": 4.800169965087934e-07,
|
|
"loss": 3.1586430850438774e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 556,
|
|
"train_speed(iter/s)": 0.010006
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 962.0,
|
|
"completions/mean_length": 320.171875,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.3229921716439548,
|
|
"grad_norm": 0.5314422694139194,
|
|
"kl": 0.0333251953125,
|
|
"learning_rate": 4.799298183517618e-07,
|
|
"loss": 3.324791032355279e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3515625,
|
|
"reward_std": 0.2907869815826416,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 557,
|
|
"train_speed(iter/s)": 0.010018
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 782.0,
|
|
"completions/mean_length": 324.1328125,
|
|
"completions/min_length": 133.0,
|
|
"epoch": 0.3235720498695274,
|
|
"grad_norm": 0.5123561260648561,
|
|
"kl": 0.0313720703125,
|
|
"learning_rate": 4.798424593223178e-07,
|
|
"loss": 3.133242717012763e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.2963140904903412,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 558,
|
|
"train_speed(iter/s)": 0.010029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1029.0,
|
|
"completions/mean_length": 338.8515625,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.3241519280951,
|
|
"grad_norm": 0.4505156909570555,
|
|
"kl": 0.02972412109375,
|
|
"learning_rate": 4.797549194975649e-07,
|
|
"loss": 2.9727756555075757e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.2494390904903412,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 559,
|
|
"train_speed(iter/s)": 0.010039
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1199.0,
|
|
"completions/mean_length": 314.7734375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.32473180632067267,
|
|
"grad_norm": 0.5415624207592917,
|
|
"kl": 0.031982421875,
|
|
"learning_rate": 4.796671989547666e-07,
|
|
"loss": 3.197913247277029e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.323806494474411,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 560,
|
|
"train_speed(iter/s)": 0.010049
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1543.0,
|
|
"completions/mean_length": 352.5546875,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.32531168454624526,
|
|
"grad_norm": 0.3840679362008074,
|
|
"kl": 0.02838134765625,
|
|
"learning_rate": 4.795792977713456e-07,
|
|
"loss": 2.835360101016704e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66796875,
|
|
"reward_std": 0.2067507952451706,
|
|
"rewards/CSTORM/mean": 0.37109375,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 561,
|
|
"train_speed(iter/s)": 0.010058
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1133.0,
|
|
"completions/mean_length": 360.671875,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.3258915627718179,
|
|
"grad_norm": 0.4247671702884637,
|
|
"kl": 0.02557373046875,
|
|
"learning_rate": 4.79491216024884e-07,
|
|
"loss": 2.5546432880219072e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.2728765904903412,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 562,
|
|
"train_speed(iter/s)": 0.010066
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 802.0,
|
|
"completions/mean_length": 335.0234375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.32647144099739056,
|
|
"grad_norm": 0.4377640637553353,
|
|
"kl": 0.030517578125,
|
|
"learning_rate": 4.794029537931233e-07,
|
|
"loss": 3.0527902708854526e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.19397208094596863,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 563,
|
|
"train_speed(iter/s)": 0.010078
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 945.0,
|
|
"completions/mean_length": 361.53125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.32705131922296315,
|
|
"grad_norm": 0.5472834833034336,
|
|
"kl": 0.02850341796875,
|
|
"learning_rate": 4.793145111539643e-07,
|
|
"loss": 2.850618511729408e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.30021828413009644,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 564,
|
|
"train_speed(iter/s)": 0.010089
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 741.0,
|
|
"completions/mean_length": 344.0546875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.3276311974485358,
|
|
"grad_norm": 0.5515223515244753,
|
|
"kl": 0.03179931640625,
|
|
"learning_rate": 4.792258881854671e-07,
|
|
"loss": 3.1789488275535405e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.41239455342292786,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 565,
|
|
"train_speed(iter/s)": 0.0101
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1195.0,
|
|
"completions/mean_length": 338.8125,
|
|
"completions/min_length": 135.0,
|
|
"epoch": 0.32821107567410845,
|
|
"grad_norm": 0.4678698888891441,
|
|
"kl": 0.03076171875,
|
|
"learning_rate": 4.79137084965851e-07,
|
|
"loss": 3.0726023396709934e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3515625,
|
|
"reward_std": 0.2552446126937866,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 566,
|
|
"train_speed(iter/s)": 0.010096
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1826.0,
|
|
"completions/mean_length": 377.03125,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.32879095389968105,
|
|
"grad_norm": 0.4417405098451137,
|
|
"kl": 0.0296630859375,
|
|
"learning_rate": 4.790481015734942e-07,
|
|
"loss": 2.9672988603124395e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37109375,
|
|
"reward_std": 0.15974397957324982,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 567,
|
|
"train_speed(iter/s)": 0.010103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 817.0,
|
|
"completions/mean_length": 325.4765625,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.3293708321252537,
|
|
"grad_norm": 0.5156151503471298,
|
|
"kl": 0.029296875,
|
|
"learning_rate": 4.789589380869342e-07,
|
|
"loss": 2.930143818957731e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.69140625,
|
|
"reward_std": 0.3076205551624298,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 568,
|
|
"train_speed(iter/s)": 0.010115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 839.0,
|
|
"completions/mean_length": 345.4921875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.32995071035082635,
|
|
"grad_norm": 0.42946230632995275,
|
|
"kl": 0.160400390625,
|
|
"learning_rate": 4.78869594584867e-07,
|
|
"loss": 0.0001599853567313403,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.24952585995197296,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 569,
|
|
"train_speed(iter/s)": 0.010127
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 843.0,
|
|
"completions/mean_length": 329.8828125,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.33053058857639894,
|
|
"grad_norm": 0.5551100088161949,
|
|
"kl": 0.03143310546875,
|
|
"learning_rate": 4.78780071146148e-07,
|
|
"loss": 3.143022331641987e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.24071526527404785,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 570,
|
|
"train_speed(iter/s)": 0.010139
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1013.0,
|
|
"completions/mean_length": 337.578125,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.3311104668019716,
|
|
"grad_norm": 0.4469654746254078,
|
|
"kl": 0.0787353515625,
|
|
"learning_rate": 4.786903678497911e-07,
|
|
"loss": 7.917211769381538e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.2728765904903412,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 571,
|
|
"train_speed(iter/s)": 0.01015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 891.0,
|
|
"completions/mean_length": 332.7109375,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.33169034502754424,
|
|
"grad_norm": 0.5122408843982361,
|
|
"kl": 0.032958984375,
|
|
"learning_rate": 4.786004847749691e-07,
|
|
"loss": 3.2924592233030125e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.24568146467208862,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 572,
|
|
"train_speed(iter/s)": 0.010161
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 789.0,
|
|
"completions/mean_length": 329.5625,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.33227022325311684,
|
|
"grad_norm": 0.5223901070798942,
|
|
"kl": 0.032470703125,
|
|
"learning_rate": 4.785104220010132e-07,
|
|
"loss": 3.235990152461454e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.35007143020629883,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 573,
|
|
"train_speed(iter/s)": 0.010174
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/mean_length": 372.1171875,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.3328501014786895,
|
|
"grad_norm": 0.5236268502061613,
|
|
"kl": 0.0318603515625,
|
|
"learning_rate": 4.784201796074137e-07,
|
|
"loss": 3.184100205544382e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.3185577690601349,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 574,
|
|
"train_speed(iter/s)": 0.010183
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 869.0,
|
|
"completions/mean_length": 360.7265625,
|
|
"completions/min_length": 132.0,
|
|
"epoch": 0.3334299797042621,
|
|
"grad_norm": 0.49890290798690257,
|
|
"kl": 0.02825927734375,
|
|
"learning_rate": 4.783297576738188e-07,
|
|
"loss": 2.8259943064767867e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.20178458094596863,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 575,
|
|
"train_speed(iter/s)": 0.010193
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1201.0,
|
|
"completions/mean_length": 353.09375,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.33400985792983473,
|
|
"grad_norm": 0.44480652362844675,
|
|
"kl": 0.04412841796875,
|
|
"learning_rate": 4.782391562800357e-07,
|
|
"loss": 4.413198257680051e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.1911257952451706,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 576,
|
|
"train_speed(iter/s)": 0.010203
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 794.0,
|
|
"completions/mean_length": 336.1484375,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.3345897361554074,
|
|
"grad_norm": 0.42956980199251665,
|
|
"kl": 0.02996826171875,
|
|
"learning_rate": 4.781483755060297e-07,
|
|
"loss": 2.9974416975164786e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.2536258101463318,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 577,
|
|
"train_speed(iter/s)": 0.010214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 644.0,
|
|
"completions/mean_length": 325.734375,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.33516961438098,
|
|
"grad_norm": 0.4198860598264259,
|
|
"kl": 0.03240966796875,
|
|
"learning_rate": 4.780574154319246e-07,
|
|
"loss": 3.2425185054307804e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.19233438372612,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 578,
|
|
"train_speed(iter/s)": 0.010227
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 671.0,
|
|
"completions/mean_length": 339.015625,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.3357494926065526,
|
|
"grad_norm": 0.5574645146113528,
|
|
"kl": 0.02935791015625,
|
|
"learning_rate": 4.779662761380023e-07,
|
|
"loss": 2.9326773073989898e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.3857266902923584,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 579,
|
|
"train_speed(iter/s)": 0.010239
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 707.0,
|
|
"completions/mean_length": 334.6015625,
|
|
"completions/min_length": 137.0,
|
|
"epoch": 0.3363293708321253,
|
|
"grad_norm": 0.45300973256921434,
|
|
"kl": 0.03253173828125,
|
|
"learning_rate": 4.778749577047029e-07,
|
|
"loss": 3.247672793804668e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.2607455849647522,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 580,
|
|
"train_speed(iter/s)": 0.01025
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 595.0,
|
|
"completions/mean_length": 312.1328125,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.33690924905769787,
|
|
"grad_norm": 33.095141412473694,
|
|
"kl": 49.2664794921875,
|
|
"learning_rate": 4.777834602126248e-07,
|
|
"loss": 0.04934834688901901,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.234375,
|
|
"reward_std": 0.349100261926651,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 581,
|
|
"train_speed(iter/s)": 0.010185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1212.0,
|
|
"completions/mean_length": 329.671875,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.3374891272832705,
|
|
"grad_norm": 0.5765822853312307,
|
|
"kl": 0.0274658203125,
|
|
"learning_rate": 4.776917837425242e-07,
|
|
"loss": 2.744896301010158e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.3490813970565796,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 582,
|
|
"train_speed(iter/s)": 0.010195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 823.0,
|
|
"completions/mean_length": 321.859375,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.33806900550884317,
|
|
"grad_norm": 0.4604441250422366,
|
|
"kl": 0.033935546875,
|
|
"learning_rate": 4.775999283753154e-07,
|
|
"loss": 3.392400685697794e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.24069640040397644,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 583,
|
|
"train_speed(iter/s)": 0.010207
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 964.0,
|
|
"completions/mean_length": 346.703125,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.33864888373441576,
|
|
"grad_norm": 0.40166012574668547,
|
|
"kl": 0.03021240234375,
|
|
"learning_rate": 4.775078941920704e-07,
|
|
"loss": 3.0222887289710343e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.1765587031841278,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 584,
|
|
"train_speed(iter/s)": 0.010218
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1484.0,
|
|
"completions/mean_length": 349.796875,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.3392287619599884,
|
|
"grad_norm": 0.5205541882259258,
|
|
"kl": 0.033935546875,
|
|
"learning_rate": 4.774156812740194e-07,
|
|
"loss": 3.3897180401254445e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.26225143671035767,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 585,
|
|
"train_speed(iter/s)": 0.01021
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 573.0,
|
|
"completions/mean_length": 316.625,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.339808640185561,
|
|
"grad_norm": 0.44990559077449616,
|
|
"kl": 0.0338134765625,
|
|
"learning_rate": 4.773232897025499e-07,
|
|
"loss": 3.380005364306271e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.19298207759857178,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 586,
|
|
"train_speed(iter/s)": 0.010223
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/mean_length": 341.390625,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.34038851841113366,
|
|
"grad_norm": 0.5729483055431439,
|
|
"kl": 0.030029296875,
|
|
"learning_rate": 4.772307195592076e-07,
|
|
"loss": 3.0000959668541327e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.3390023708343506,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 587,
|
|
"train_speed(iter/s)": 0.010234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1420.0,
|
|
"completions/mean_length": 320.890625,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.3409683966367063,
|
|
"grad_norm": 0.5493663522242002,
|
|
"kl": 0.02825927734375,
|
|
"learning_rate": 4.771379709256952e-07,
|
|
"loss": 2.825208503054455e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.2751619815826416,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 588,
|
|
"train_speed(iter/s)": 0.010243
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 927.0,
|
|
"completions/mean_length": 313.921875,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.3415482748622789,
|
|
"grad_norm": 0.4102119959915088,
|
|
"kl": 0.0303955078125,
|
|
"learning_rate": 4.770450438838736e-07,
|
|
"loss": 3.0349478038260713e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.18033519387245178,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 589,
|
|
"train_speed(iter/s)": 0.010241
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 898.0,
|
|
"completions/mean_length": 373.0625,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.34212815308785155,
|
|
"grad_norm": 0.43721327963630846,
|
|
"kl": 0.02752685546875,
|
|
"learning_rate": 4.769519385157605e-07,
|
|
"loss": 2.7528531063580886e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.262345552444458,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 590,
|
|
"train_speed(iter/s)": 0.010252
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1002.0,
|
|
"completions/mean_length": 337.09375,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.3427080313134242,
|
|
"grad_norm": 0.38624496962843485,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 4.768586549035314e-07,
|
|
"loss": 3.280969394836575e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.640625,
|
|
"reward_std": 0.26223257184028625,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 591,
|
|
"train_speed(iter/s)": 0.010263
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 703.0,
|
|
"completions/mean_length": 337.8984375,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.3432879095389968,
|
|
"grad_norm": 0.5600172558058419,
|
|
"kl": 0.0306396484375,
|
|
"learning_rate": 4.7676519312951915e-07,
|
|
"loss": 3.0641585908597335e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.3406212329864502,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 592,
|
|
"train_speed(iter/s)": 0.010274
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 896.0,
|
|
"completions/mean_length": 310.4765625,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.34386778776456944,
|
|
"grad_norm": 0.44927198292620213,
|
|
"kl": 0.037353515625,
|
|
"learning_rate": 4.7667155327621365e-07,
|
|
"loss": 3.736088183359243e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.18788404762744904,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 593,
|
|
"train_speed(iter/s)": 0.010285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 863.0,
|
|
"completions/mean_length": 342.3515625,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.3444476659901421,
|
|
"grad_norm": 0.5713832906024858,
|
|
"kl": 0.0462646484375,
|
|
"learning_rate": 4.765777354262621e-07,
|
|
"loss": 4.6298373490571976e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.3678351640701294,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 594,
|
|
"train_speed(iter/s)": 0.010296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 553.0,
|
|
"completions/mean_length": 297.4140625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.3450275442157147,
|
|
"grad_norm": 0.4535564985906918,
|
|
"kl": 0.033203125,
|
|
"learning_rate": 4.764837396624687e-07,
|
|
"loss": 3.3142201573355123e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.67578125,
|
|
"reward_std": 0.1911257952451706,
|
|
"rewards/CSTORM/mean": 0.38671875,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 595,
|
|
"train_speed(iter/s)": 0.010309
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 823.0,
|
|
"completions/mean_length": 307.21875,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.34560742244128734,
|
|
"grad_norm": 0.5316048748810976,
|
|
"kl": 0.03155517578125,
|
|
"learning_rate": 4.763895660677947e-07,
|
|
"loss": 3.154604200972244e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.27019575238227844,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 596,
|
|
"train_speed(iter/s)": 0.010318
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1043.0,
|
|
"completions/mean_length": 328.3046875,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.34618730066685993,
|
|
"grad_norm": 0.5847065778087561,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 4.7629521472535846e-07,
|
|
"loss": 3.284422928118147e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.4260166585445404,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 597,
|
|
"train_speed(iter/s)": 0.010328
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 888.0,
|
|
"completions/mean_length": 322.6796875,
|
|
"completions/min_length": 61.0,
|
|
"epoch": 0.3467671788924326,
|
|
"grad_norm": 0.295477205656884,
|
|
"kl": 0.031494140625,
|
|
"learning_rate": 4.76200685718435e-07,
|
|
"loss": 3.155102604068816e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.08835469186306,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 598,
|
|
"train_speed(iter/s)": 0.010339
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 691.0,
|
|
"completions/mean_length": 338.5859375,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.34734705711800523,
|
|
"grad_norm": 0.39538127235738596,
|
|
"kl": 0.029541015625,
|
|
"learning_rate": 4.7610597913045633e-07,
|
|
"loss": 2.9442864615703e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.24809867143630981,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 599,
|
|
"train_speed(iter/s)": 0.010351
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 669.0,
|
|
"completions/mean_length": 316.6328125,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.3479269353435778,
|
|
"grad_norm": 0.4159455714694927,
|
|
"kl": 0.0335693359375,
|
|
"learning_rate": 4.7601109504501095e-07,
|
|
"loss": 3.360648770467378e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.220455601811409,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 600,
|
|
"train_speed(iter/s)": 0.010362
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 651.0,
|
|
"completions/mean_length": 344.890625,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.3485068135691505,
|
|
"grad_norm": 0.44555038285401943,
|
|
"kl": 0.027587890625,
|
|
"learning_rate": 4.759160335458444e-07,
|
|
"loss": 2.7629819669527933e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.2698984742164612,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 601,
|
|
"train_speed(iter/s)": 0.010369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1015.0,
|
|
"completions/mean_length": 350.34375,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.3490866917947231,
|
|
"grad_norm": 0.4776682391006732,
|
|
"kl": 0.0335693359375,
|
|
"learning_rate": 4.758207947168584e-07,
|
|
"loss": 3.3538948628120124e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2109375,
|
|
"reward_std": 0.25038406252861023,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 602,
|
|
"train_speed(iter/s)": 0.010379
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/mean_length": 330.7890625,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.3496665700202957,
|
|
"grad_norm": 0.45684509130993867,
|
|
"kl": 0.035888671875,
|
|
"learning_rate": 4.757253786421115e-07,
|
|
"loss": 3.594918962335214e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.29937899112701416,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 603,
|
|
"train_speed(iter/s)": 0.010377
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 983.0,
|
|
"completions/mean_length": 348.609375,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.35024644824586837,
|
|
"grad_norm": 0.3690413185189703,
|
|
"kl": 0.03900146484375,
|
|
"learning_rate": 4.756297854058184e-07,
|
|
"loss": 3.901517629856244e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.17373128235340118,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 604,
|
|
"train_speed(iter/s)": 0.010388
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 662.0,
|
|
"completions/mean_length": 315.4453125,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.350826326471441,
|
|
"grad_norm": 0.38981865222084033,
|
|
"kl": 0.0341796875,
|
|
"learning_rate": 4.755340150923505e-07,
|
|
"loss": 3.4210992453154176e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.1676882952451706,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 605,
|
|
"train_speed(iter/s)": 0.010399
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 695.0,
|
|
"completions/mean_length": 319.421875,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.3514062046970136,
|
|
"grad_norm": 0.6060652718489487,
|
|
"kl": 0.0469970703125,
|
|
"learning_rate": 4.754380677862352e-07,
|
|
"loss": 4.693626397056505e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.36878013610839844,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 606,
|
|
"train_speed(iter/s)": 0.010411
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1063.0,
|
|
"completions/mean_length": 335.6484375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.35198608292258626,
|
|
"grad_norm": 0.48562012105352187,
|
|
"kl": 0.0333251953125,
|
|
"learning_rate": 4.753419435721563e-07,
|
|
"loss": 3.329939499963075e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.2416265904903412,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 607,
|
|
"train_speed(iter/s)": 0.010421
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 911.0,
|
|
"completions/mean_length": 319.53125,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.3525659611481589,
|
|
"grad_norm": 0.3445803645031384,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 4.7524564253495365e-07,
|
|
"loss": 3.277779251220636e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.09858439117670059,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 608,
|
|
"train_speed(iter/s)": 0.010429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 597.0,
|
|
"completions/mean_length": 295.0703125,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.3531458393737315,
|
|
"grad_norm": 0.5286733527535497,
|
|
"kl": 0.0390625,
|
|
"learning_rate": 4.7514916475962307e-07,
|
|
"loss": 3.91199100704398e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.2632945775985718,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 609,
|
|
"train_speed(iter/s)": 0.01044
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 765.0,
|
|
"completions/mean_length": 330.9765625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.35372571759930416,
|
|
"grad_norm": 0.5399509820169766,
|
|
"kl": 0.0457763671875,
|
|
"learning_rate": 4.7505251033131663e-07,
|
|
"loss": 4.5850814785808325e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.3077523708343506,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 610,
|
|
"train_speed(iter/s)": 0.01045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 676.0,
|
|
"completions/mean_length": 321.671875,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.35430559582487675,
|
|
"grad_norm": 0.5171231072811566,
|
|
"kl": 0.03271484375,
|
|
"learning_rate": 4.7495567933534215e-07,
|
|
"loss": 3.264076804043725e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.2102447748184204,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 611,
|
|
"train_speed(iter/s)": 0.01046
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1598.0,
|
|
"completions/mean_length": 349.484375,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.3548854740504494,
|
|
"grad_norm": 0.4121082740010195,
|
|
"kl": 0.0340576171875,
|
|
"learning_rate": 4.748586718571632e-07,
|
|
"loss": 3.401865978958085e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.2145632952451706,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 612,
|
|
"train_speed(iter/s)": 0.010468
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1622.0,
|
|
"completions/mean_length": 320.9375,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.35546535227602205,
|
|
"grad_norm": 0.6326789956830571,
|
|
"kl": 0.0369873046875,
|
|
"learning_rate": 4.7476148798239934e-07,
|
|
"loss": 3.700035085785203e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.3754970133304596,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 613,
|
|
"train_speed(iter/s)": 0.010475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 620.0,
|
|
"completions/mean_length": 293.453125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.35604523050159465,
|
|
"grad_norm": 0.5272165078098755,
|
|
"kl": 0.0416259765625,
|
|
"learning_rate": 4.746641277968256e-07,
|
|
"loss": 4.156929935561493e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.3275640904903412,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 614,
|
|
"train_speed(iter/s)": 0.010487
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1040.0,
|
|
"completions/mean_length": 319.390625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.3566251087271673,
|
|
"grad_norm": 0.5923584243884282,
|
|
"kl": 0.037109375,
|
|
"learning_rate": 4.7456659138637277e-07,
|
|
"loss": 3.7066227378090844e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.28515625,
|
|
"reward_std": 0.3333245813846588,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 615,
|
|
"train_speed(iter/s)": 0.010497
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 892.0,
|
|
"completions/mean_length": 300.015625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.35720498695273994,
|
|
"grad_norm": 0.6125734184528243,
|
|
"kl": 0.036865234375,
|
|
"learning_rate": 4.7446887883712726e-07,
|
|
"loss": 3.6870296753477305e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.3682642877101898,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 616,
|
|
"train_speed(iter/s)": 0.010507
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 637.0,
|
|
"completions/mean_length": 299.3984375,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.35778486517831254,
|
|
"grad_norm": 0.5764004041215463,
|
|
"kl": 0.040283203125,
|
|
"learning_rate": 4.7437099023533055e-07,
|
|
"loss": 4.0263264963869005e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.2686898708343506,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 617,
|
|
"train_speed(iter/s)": 0.010519
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 541.0,
|
|
"completions/mean_length": 304.2578125,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.3583647434038852,
|
|
"grad_norm": 0.48472139270477443,
|
|
"kl": 0.037353515625,
|
|
"learning_rate": 4.742729256673799e-07,
|
|
"loss": 3.73552757082507e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6640625,
|
|
"reward_std": 0.21762818098068237,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 618,
|
|
"train_speed(iter/s)": 0.010531
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1319.0,
|
|
"completions/mean_length": 337.5546875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.35894462162945784,
|
|
"grad_norm": 0.48736844394216294,
|
|
"kl": 0.03515625,
|
|
"learning_rate": 4.7417468521982783e-07,
|
|
"loss": 3.523399573168717e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.2258697748184204,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 619,
|
|
"train_speed(iter/s)": 0.01054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 761.0,
|
|
"completions/mean_length": 289.90625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.35952449985503043,
|
|
"grad_norm": 0.637588841371175,
|
|
"kl": 0.03955078125,
|
|
"learning_rate": 4.740762689793819e-07,
|
|
"loss": 3.9589329389855266e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33984375,
|
|
"reward_std": 0.3298494815826416,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 620,
|
|
"train_speed(iter/s)": 0.010551
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/mean_length": 352.03125,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.3601043780806031,
|
|
"grad_norm": 0.5695978461927517,
|
|
"kl": 0.0338134765625,
|
|
"learning_rate": 4.739776770329049e-07,
|
|
"loss": 3.3803215046646073e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.3155648708343506,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 621,
|
|
"train_speed(iter/s)": 0.010558
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 592.0,
|
|
"completions/mean_length": 295.0,
|
|
"completions/min_length": 71.0,
|
|
"epoch": 0.3606842563061757,
|
|
"grad_norm": 0.5794965036502868,
|
|
"kl": 0.0408935546875,
|
|
"learning_rate": 4.738789094674149e-07,
|
|
"loss": 4.0915561839938164e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.33994734287261963,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 622,
|
|
"train_speed(iter/s)": 0.010569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 771.0,
|
|
"completions/mean_length": 294.4453125,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.3612641345317483,
|
|
"grad_norm": 0.5257012804015695,
|
|
"kl": 0.0367431640625,
|
|
"learning_rate": 4.7377996637008466e-07,
|
|
"loss": 3.668406498036347e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.2801281809806824,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 623,
|
|
"train_speed(iter/s)": 0.01058
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 688.0,
|
|
"completions/mean_length": 328.5078125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.361844012757321,
|
|
"grad_norm": 0.5245454016743755,
|
|
"kl": 0.0369873046875,
|
|
"learning_rate": 4.7368084782824204e-07,
|
|
"loss": 3.702555113704875e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.2878088653087616,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 624,
|
|
"train_speed(iter/s)": 0.010591
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 899.0,
|
|
"completions/mean_length": 336.421875,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.36242389098289357,
|
|
"grad_norm": 0.6470247960479665,
|
|
"kl": 0.066162109375,
|
|
"learning_rate": 4.735815539293697e-07,
|
|
"loss": 6.606581882806495e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2890625,
|
|
"reward_std": 0.4133697748184204,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 625,
|
|
"train_speed(iter/s)": 0.010601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1692.0,
|
|
"completions/mean_length": 332.8046875,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.3630037692084662,
|
|
"grad_norm": 0.5974439182658295,
|
|
"kl": 0.03155517578125,
|
|
"learning_rate": 4.7348208476110523e-07,
|
|
"loss": 3.153822763124481e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.25390625,
|
|
"reward_std": 0.4195445775985718,
|
|
"rewards/CSTORM/mean": 0.23046875,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 626,
|
|
"train_speed(iter/s)": 0.010608
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 757.0,
|
|
"completions/mean_length": 284.609375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.36358364743403887,
|
|
"grad_norm": 0.5482376957757044,
|
|
"kl": 0.1312255859375,
|
|
"learning_rate": 4.733824404112406e-07,
|
|
"loss": 0.00013109076826367527,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.3032831847667694,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 627,
|
|
"train_speed(iter/s)": 0.010619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 803.0,
|
|
"completions/mean_length": 293.0546875,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.36416352565961146,
|
|
"grad_norm": 0.5807176169623153,
|
|
"kl": 0.0379638671875,
|
|
"learning_rate": 4.7328262096772264e-07,
|
|
"loss": 3.7891255487920716e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.3419354557991028,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 628,
|
|
"train_speed(iter/s)": 0.01063
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/mean_length": 275.265625,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.3647434038851841,
|
|
"grad_norm": 0.44928639090524747,
|
|
"kl": 0.085205078125,
|
|
"learning_rate": 4.731826265186527e-07,
|
|
"loss": 8.505606092512608e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.18318147957324982,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 629,
|
|
"train_speed(iter/s)": 0.010641
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1150.0,
|
|
"completions/mean_length": 321.4921875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.36532328211075676,
|
|
"grad_norm": 0.4833782274943525,
|
|
"kl": 0.0384521484375,
|
|
"learning_rate": 4.730824571522864e-07,
|
|
"loss": 3.844549792120233e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.21861818432807922,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 630,
|
|
"train_speed(iter/s)": 0.010651
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 668.0,
|
|
"completions/mean_length": 309.5625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.36590316033632936,
|
|
"grad_norm": 0.5786434866800213,
|
|
"kl": 0.039794921875,
|
|
"learning_rate": 4.729821129570341e-07,
|
|
"loss": 3.9711754652671516e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.29519227147102356,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 631,
|
|
"train_speed(iter/s)": 0.010662
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 798.0,
|
|
"completions/mean_length": 332.8515625,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.366483038561902,
|
|
"grad_norm": 0.5294839084589025,
|
|
"kl": 0.034912109375,
|
|
"learning_rate": 4.7288159402146e-07,
|
|
"loss": 3.485654087853618e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.3508697748184204,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 632,
|
|
"train_speed(iter/s)": 0.010673
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1051.0,
|
|
"completions/mean_length": 306.359375,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.36706291678747466,
|
|
"grad_norm": 0.5635841234905964,
|
|
"kl": 0.036376953125,
|
|
"learning_rate": 4.7278090043428296e-07,
|
|
"loss": 3.6464691220317036e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.31435626745224,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 633,
|
|
"train_speed(iter/s)": 0.010683
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1007.0,
|
|
"completions/mean_length": 311.0078125,
|
|
"completions/min_length": 136.0,
|
|
"epoch": 0.36764279501304725,
|
|
"grad_norm": 0.5077866514860907,
|
|
"kl": 0.1026611328125,
|
|
"learning_rate": 4.7268003228437583e-07,
|
|
"loss": 0.00010233785724267364,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.74609375,
|
|
"reward_std": 0.2517244815826416,
|
|
"rewards/CSTORM/mean": 0.3984375,
|
|
"rewards/CSTORM/std": 0.20195281505584717,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.8515625,
|
|
"rewards/VQAORM/std": 0.356930136680603,
|
|
"step": 634,
|
|
"train_speed(iter/s)": 0.010676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 572.0,
|
|
"completions/mean_length": 301.234375,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.3682226732386199,
|
|
"grad_norm": 0.49900597349219183,
|
|
"kl": 0.0347900390625,
|
|
"learning_rate": 4.725789896607653e-07,
|
|
"loss": 3.4703523851931095e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.19503000378608704,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 635,
|
|
"train_speed(iter/s)": 0.010687
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 955.0,
|
|
"completions/mean_length": 286.015625,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.3688025514641925,
|
|
"grad_norm": 7.6743215026101534,
|
|
"kl": 0.0374755859375,
|
|
"learning_rate": 4.724777726526325e-07,
|
|
"loss": 3.7383666494861245e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.2384016364812851,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 636,
|
|
"train_speed(iter/s)": 0.010623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 615.0,
|
|
"completions/mean_length": 293.8046875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.36938242968976515,
|
|
"grad_norm": 0.42148449281307804,
|
|
"kl": 0.038330078125,
|
|
"learning_rate": 4.7237638134931205e-07,
|
|
"loss": 3.834480594377965e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.2547025680541992,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 637,
|
|
"train_speed(iter/s)": 0.010634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1253.0,
|
|
"completions/mean_length": 322.09375,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.3699623079153378,
|
|
"grad_norm": 0.5833492142273231,
|
|
"kl": 0.0355224609375,
|
|
"learning_rate": 4.722748158402927e-07,
|
|
"loss": 3.550307883415371e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.3376619815826416,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 638,
|
|
"train_speed(iter/s)": 0.010643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 819.0,
|
|
"completions/mean_length": 293.296875,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.3705421861409104,
|
|
"grad_norm": 0.5655605595353634,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 4.721730762152168e-07,
|
|
"loss": 3.797889803536236e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.3609299063682556,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 639,
|
|
"train_speed(iter/s)": 0.010653
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1077.0,
|
|
"completions/mean_length": 317.03125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.37112206436648304,
|
|
"grad_norm": 0.4461425604226927,
|
|
"kl": 0.0379638671875,
|
|
"learning_rate": 4.720711625638803e-07,
|
|
"loss": 3.806900713243522e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.54296875,
|
|
"reward_std": 0.278787761926651,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 640,
|
|
"train_speed(iter/s)": 0.010663
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 535.0,
|
|
"completions/mean_length": 286.4765625,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.3717019425920557,
|
|
"grad_norm": 0.46887000740818674,
|
|
"kl": 0.040771484375,
|
|
"learning_rate": 4.71969074976233e-07,
|
|
"loss": 4.084004831383936e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6640625,
|
|
"reward_std": 0.230056494474411,
|
|
"rewards/CSTORM/mean": 0.375,
|
|
"rewards/CSTORM/std": 0.2173570692539215,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 641,
|
|
"train_speed(iter/s)": 0.010674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1146.0,
|
|
"completions/mean_length": 323.109375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.3722818208176283,
|
|
"grad_norm": 0.6112261607881492,
|
|
"kl": 0.042236328125,
|
|
"learning_rate": 4.718668135423781e-07,
|
|
"loss": 4.2164283513557166e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2578125,
|
|
"reward_std": 0.40330514311790466,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 642,
|
|
"train_speed(iter/s)": 0.010682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1600.0,
|
|
"completions/mean_length": 317.5625,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.37286169904320093,
|
|
"grad_norm": 0.4880000030961268,
|
|
"kl": 0.04052734375,
|
|
"learning_rate": 4.717643783525722e-07,
|
|
"loss": 4.0509916289011016e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.34746256470680237,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 643,
|
|
"train_speed(iter/s)": 0.010689
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 822.0,
|
|
"completions/mean_length": 317.2265625,
|
|
"completions/min_length": 123.0,
|
|
"epoch": 0.3734415772687736,
|
|
"grad_norm": 0.4461989476550787,
|
|
"kl": 0.032958984375,
|
|
"learning_rate": 4.716617694972252e-07,
|
|
"loss": 3.2981435651890934e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.32281649112701416,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 644,
|
|
"train_speed(iter/s)": 0.010697
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2235.0,
|
|
"completions/mean_length": 352.1328125,
|
|
"completions/min_length": 126.0,
|
|
"epoch": 0.3740214554943462,
|
|
"grad_norm": 0.5269923142618319,
|
|
"kl": 0.0362548828125,
|
|
"learning_rate": 4.715589870669004e-07,
|
|
"loss": 3.6123972677160054e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.2921273708343506,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 645,
|
|
"train_speed(iter/s)": 0.010701
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 698.0,
|
|
"completions/mean_length": 315.5859375,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.3746013337199188,
|
|
"grad_norm": 0.5222972985462799,
|
|
"kl": 0.035400390625,
|
|
"learning_rate": 4.714560311523143e-07,
|
|
"loss": 3.539887256920338e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2728765904903412,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 646,
|
|
"train_speed(iter/s)": 0.010713
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 742.0,
|
|
"completions/mean_length": 290.9296875,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.3751812119454914,
|
|
"grad_norm": 0.5380025513385289,
|
|
"kl": 0.038818359375,
|
|
"learning_rate": 4.7135290184433624e-07,
|
|
"loss": 3.88166299671866e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.22664926946163177,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 647,
|
|
"train_speed(iter/s)": 0.010724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 622.0,
|
|
"completions/mean_length": 318.5625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.37576109017106407,
|
|
"grad_norm": 0.523766965930979,
|
|
"kl": 0.0333251953125,
|
|
"learning_rate": 4.712495992339892e-07,
|
|
"loss": 3.326336809550412e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.297906756401062,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 648,
|
|
"train_speed(iter/s)": 0.010736
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 857.0,
|
|
"completions/mean_length": 302.46875,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.3763409683966367,
|
|
"grad_norm": 0.45795075538234503,
|
|
"kl": 0.039794921875,
|
|
"learning_rate": 4.7114612341244845e-07,
|
|
"loss": 3.988540265709162e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.211453378200531,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 649,
|
|
"train_speed(iter/s)": 0.010746
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 685.0,
|
|
"completions/mean_length": 305.6953125,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.3769208466222093,
|
|
"grad_norm": 0.4683205715703724,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 4.710424744710426e-07,
|
|
"loss": 3.962846676586196e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.17778617143630981,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 650,
|
|
"train_speed(iter/s)": 0.010757
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 744.0,
|
|
"completions/mean_length": 312.3984375,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.37750072484778197,
|
|
"grad_norm": 0.42952121796535536,
|
|
"kl": 0.0401611328125,
|
|
"learning_rate": 4.7093865250125275e-07,
|
|
"loss": 4.0232356695923954e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.1442507952451706,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 651,
|
|
"train_speed(iter/s)": 0.010766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1356.0,
|
|
"completions/mean_length": 335.4140625,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.3780806030733546,
|
|
"grad_norm": 0.4876176540728279,
|
|
"kl": 0.08544921875,
|
|
"learning_rate": 4.7083465759471305e-07,
|
|
"loss": 8.570826321374625e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.269118994474411,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 652,
|
|
"train_speed(iter/s)": 0.010716
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 579.0,
|
|
"completions/mean_length": 284.09375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.3786604812989272,
|
|
"grad_norm": 0.5386344135773785,
|
|
"kl": 0.0394287109375,
|
|
"learning_rate": 4.707304898432098e-07,
|
|
"loss": 3.938022564398125e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.21443147957324982,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 653,
|
|
"train_speed(iter/s)": 0.010727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 823.0,
|
|
"completions/mean_length": 299.875,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.37924035952449986,
|
|
"grad_norm": 0.35543368484890187,
|
|
"kl": 0.041259765625,
|
|
"learning_rate": 4.706261493386825e-07,
|
|
"loss": 4.128757427679375e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.17644576728343964,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 654,
|
|
"train_speed(iter/s)": 0.010737
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 927.0,
|
|
"completions/mean_length": 300.078125,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.3798202377500725,
|
|
"grad_norm": 0.49794643236675956,
|
|
"kl": 0.03759765625,
|
|
"learning_rate": 4.705216361732226e-07,
|
|
"loss": 3.7651123420801014e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62109375,
|
|
"reward_std": 0.24525238573551178,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 655,
|
|
"train_speed(iter/s)": 0.010747
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 597.0,
|
|
"completions/mean_length": 298.90625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.3804001159756451,
|
|
"grad_norm": 0.5823564819764103,
|
|
"kl": 0.0377197265625,
|
|
"learning_rate": 4.70416950439074e-07,
|
|
"loss": 3.7731268093921244e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.2482304871082306,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 656,
|
|
"train_speed(iter/s)": 0.010759
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 676.0,
|
|
"completions/mean_length": 323.9375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.38097999420121775,
|
|
"grad_norm": 0.4784431316006857,
|
|
"kl": 0.03466796875,
|
|
"learning_rate": 4.7031209222863314e-07,
|
|
"loss": 3.4654753108043224e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.57421875,
|
|
"reward_std": 0.30129510164260864,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 657,
|
|
"train_speed(iter/s)": 0.01077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1000.0,
|
|
"completions/mean_length": 318.9375,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.38155987242679035,
|
|
"grad_norm": 0.6418261978094596,
|
|
"kl": 0.038330078125,
|
|
"learning_rate": 4.7020706163444854e-07,
|
|
"loss": 3.833074151771143e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.3112463653087616,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 658,
|
|
"train_speed(iter/s)": 0.01078
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 577.0,
|
|
"completions/mean_length": 291.46875,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.382139750652363,
|
|
"grad_norm": 0.38924724071309985,
|
|
"kl": 0.0408935546875,
|
|
"learning_rate": 4.7010185874922105e-07,
|
|
"loss": 4.095198164577596e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.1442507952451706,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 659,
|
|
"train_speed(iter/s)": 0.010791
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1851.0,
|
|
"completions/mean_length": 344.4140625,
|
|
"completions/min_length": 131.0,
|
|
"epoch": 0.38271962887793565,
|
|
"grad_norm": 0.4317457461886334,
|
|
"kl": 0.0367431640625,
|
|
"learning_rate": 4.699964836658032e-07,
|
|
"loss": 3.67394823115319e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.18063247203826904,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 660,
|
|
"train_speed(iter/s)": 0.010797
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 692.0,
|
|
"completions/mean_length": 276.328125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.38329950710350824,
|
|
"grad_norm": 0.5044639799362923,
|
|
"kl": 0.04736328125,
|
|
"learning_rate": 4.6989093647719977e-07,
|
|
"loss": 4.735819311463274e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.2949736714363098,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 661,
|
|
"train_speed(iter/s)": 0.010808
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 768.0,
|
|
"completions/mean_length": 332.25,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.3838793853290809,
|
|
"grad_norm": 0.49350631529244654,
|
|
"kl": 0.03662109375,
|
|
"learning_rate": 4.697852172765676e-07,
|
|
"loss": 3.6695622839033604e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.3245859742164612,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 662,
|
|
"train_speed(iter/s)": 0.010818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1259.0,
|
|
"completions/mean_length": 343.953125,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.38445926355465354,
|
|
"grad_norm": 0.48495818156155557,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 4.6967932615721487e-07,
|
|
"loss": 3.288529842393473e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.22871607542037964,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 663,
|
|
"train_speed(iter/s)": 0.010826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 788.0,
|
|
"completions/mean_length": 333.0,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.38503914178022614,
|
|
"grad_norm": 0.5930155637092425,
|
|
"kl": 0.03564453125,
|
|
"learning_rate": 4.6957326321260206e-07,
|
|
"loss": 3.571146589820273e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.39675477147102356,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 664,
|
|
"train_speed(iter/s)": 0.010835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1142.0,
|
|
"completions/mean_length": 327.15625,
|
|
"completions/min_length": 49.0,
|
|
"epoch": 0.3856190200057988,
|
|
"grad_norm": 0.4913832314757133,
|
|
"kl": 0.0384521484375,
|
|
"learning_rate": 4.6946702853634084e-07,
|
|
"loss": 3.856192051898688e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.21762816607952118,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 665,
|
|
"train_speed(iter/s)": 0.010845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 626.0,
|
|
"completions/mean_length": 276.8984375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.38619889823137143,
|
|
"grad_norm": 0.47793813998985385,
|
|
"kl": 0.0810546875,
|
|
"learning_rate": 4.693606222221947e-07,
|
|
"loss": 8.116233948385343e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.16201049089431763,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 666,
|
|
"train_speed(iter/s)": 0.010855
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 836.0,
|
|
"completions/mean_length": 293.796875,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.38677877645694403,
|
|
"grad_norm": 0.41357425066389414,
|
|
"kl": 0.0478515625,
|
|
"learning_rate": 4.692540443640786e-07,
|
|
"loss": 4.781782627105713e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.25390625,
|
|
"reward_std": 0.20739847421646118,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 667,
|
|
"train_speed(iter/s)": 0.010866
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 856.0,
|
|
"completions/mean_length": 325.6015625,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.3873586546825167,
|
|
"grad_norm": 0.5658423014430178,
|
|
"kl": 0.037109375,
|
|
"learning_rate": 4.6914729505605873e-07,
|
|
"loss": 3.712196121341549e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.3352447748184204,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 668,
|
|
"train_speed(iter/s)": 0.010877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 847.0,
|
|
"completions/mean_length": 320.3828125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.38793853290808933,
|
|
"grad_norm": 0.5195806609365667,
|
|
"kl": 0.0400390625,
|
|
"learning_rate": 4.690403743923528e-07,
|
|
"loss": 4.0083097701426595e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.22452935576438904,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 669,
|
|
"train_speed(iter/s)": 0.010887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/mean_length": 295.421875,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.3885184111336619,
|
|
"grad_norm": 0.5353639679342264,
|
|
"kl": 0.041259765625,
|
|
"learning_rate": 4.6893328246732983e-07,
|
|
"loss": 4.1190534830093384e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.30507153272628784,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 670,
|
|
"train_speed(iter/s)": 0.010897
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1218.0,
|
|
"completions/mean_length": 305.3359375,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.3890982893592346,
|
|
"grad_norm": 0.4637032508513651,
|
|
"kl": 0.0447998046875,
|
|
"learning_rate": 4.6882601937550967e-07,
|
|
"loss": 4.487840487854555e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.2494390904903412,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 671,
|
|
"train_speed(iter/s)": 0.010905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 558.0,
|
|
"completions/mean_length": 319.671875,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.38967816758480717,
|
|
"grad_norm": 0.46435402688552124,
|
|
"kl": 0.0374755859375,
|
|
"learning_rate": 4.687185852115636e-07,
|
|
"loss": 3.749670213437639e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6953125,
|
|
"reward_std": 0.22399462759494781,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 672,
|
|
"train_speed(iter/s)": 0.010916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 753.0,
|
|
"completions/mean_length": 313.7109375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.3902580458103798,
|
|
"grad_norm": 0.4670381345175128,
|
|
"kl": 0.06103515625,
|
|
"learning_rate": 4.686109800703135e-07,
|
|
"loss": 6.115913856774569e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2522853910923004,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 673,
|
|
"train_speed(iter/s)": 0.010925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 720.0,
|
|
"completions/mean_length": 315.8203125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.39083792403595247,
|
|
"grad_norm": 0.5386286014159722,
|
|
"kl": 0.037109375,
|
|
"learning_rate": 4.6850320404673263e-07,
|
|
"loss": 3.70902635040693e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.31640625,
|
|
"reward_std": 0.3077523708343506,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 674,
|
|
"train_speed(iter/s)": 0.010935
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 636.0,
|
|
"completions/mean_length": 281.671875,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.39141780226152506,
|
|
"grad_norm": 0.4577964328053322,
|
|
"kl": 0.0400390625,
|
|
"learning_rate": 4.6839525723594456e-07,
|
|
"loss": 4.0108316170517355e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.1725226789712906,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 675,
|
|
"train_speed(iter/s)": 0.010946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1592.0,
|
|
"completions/mean_length": 318.203125,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.3919976804870977,
|
|
"grad_norm": 0.5802819620056056,
|
|
"kl": 0.0545654296875,
|
|
"learning_rate": 4.6828713973322414e-07,
|
|
"loss": 5.460865213535726e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.33006805181503296,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 676,
|
|
"train_speed(iter/s)": 0.010952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1044.0,
|
|
"completions/mean_length": 313.2734375,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.39257755871267036,
|
|
"grad_norm": 0.4332328371217218,
|
|
"kl": 0.039794921875,
|
|
"learning_rate": 4.681788516339964e-07,
|
|
"loss": 3.9716858736937866e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62890625,
|
|
"reward_std": 0.22010520100593567,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 677,
|
|
"train_speed(iter/s)": 0.010962
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1020.0,
|
|
"completions/mean_length": 310.0546875,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.39315743693824295,
|
|
"grad_norm": 0.4610607442033295,
|
|
"kl": 0.036376953125,
|
|
"learning_rate": 4.680703930338372e-07,
|
|
"loss": 3.6377743526827544e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.2024322748184204,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 678,
|
|
"train_speed(iter/s)": 0.010971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1076.0,
|
|
"completions/mean_length": 313.2578125,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.3937373151638156,
|
|
"grad_norm": 0.41599327216362375,
|
|
"kl": 0.0408935546875,
|
|
"learning_rate": 4.679617640284728e-07,
|
|
"loss": 4.0936949517345056e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.18559867143630981,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 679,
|
|
"train_speed(iter/s)": 0.01098
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 670.0,
|
|
"completions/mean_length": 324.1484375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.39431719338938825,
|
|
"grad_norm": 0.5023324043992143,
|
|
"kl": 0.037353515625,
|
|
"learning_rate": 4.678529647137799e-07,
|
|
"loss": 3.736863800440915e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.31409263610839844,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 680,
|
|
"train_speed(iter/s)": 0.010991
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 658.0,
|
|
"completions/mean_length": 298.84375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.39489707161496085,
|
|
"grad_norm": 0.524784639088043,
|
|
"kl": 0.041015625,
|
|
"learning_rate": 4.6774399518578537e-07,
|
|
"loss": 4.099362558918074e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.24646098911762238,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 681,
|
|
"train_speed(iter/s)": 0.011002
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 881.0,
|
|
"completions/mean_length": 294.703125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.3954769498405335,
|
|
"grad_norm": 0.49908657135899953,
|
|
"kl": 0.041015625,
|
|
"learning_rate": 4.676348555406667e-07,
|
|
"loss": 4.108437860850245e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.20795938372612,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 682,
|
|
"train_speed(iter/s)": 0.011012
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1744.0,
|
|
"completions/mean_length": 361.1875,
|
|
"completions/min_length": 128.0,
|
|
"epoch": 0.3960568280661061,
|
|
"grad_norm": 0.43622433267015254,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 4.6752554587475103e-07,
|
|
"loss": 3.960785033996217e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.19800812005996704,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 683,
|
|
"train_speed(iter/s)": 0.011015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 718.0,
|
|
"completions/mean_length": 294.9609375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.39663670629167874,
|
|
"grad_norm": 0.6680033103244695,
|
|
"kl": 0.0406494140625,
|
|
"learning_rate": 4.674160662845158e-07,
|
|
"loss": 4.065588291268796e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.3394314646720886,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 684,
|
|
"train_speed(iter/s)": 0.011025
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 709.0,
|
|
"completions/mean_length": 308.8515625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.3972165845172514,
|
|
"grad_norm": 0.5563059236770127,
|
|
"kl": 0.0435791015625,
|
|
"learning_rate": 4.6730641686658855e-07,
|
|
"loss": 4.360017555882223e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.31230428814888,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 685,
|
|
"train_speed(iter/s)": 0.011036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1566.0,
|
|
"completions/mean_length": 315.359375,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.397796462742824,
|
|
"grad_norm": 0.5603860336700033,
|
|
"kl": 0.041015625,
|
|
"learning_rate": 4.671965977177466e-07,
|
|
"loss": 4.097449345863424e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.26571178436279297,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 686,
|
|
"train_speed(iter/s)": 0.011042
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 611.0,
|
|
"completions/mean_length": 284.1640625,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.39837634096839664,
|
|
"grad_norm": 0.5676159300659409,
|
|
"kl": 0.041748046875,
|
|
"learning_rate": 4.6708660893491686e-07,
|
|
"loss": 4.168313171248883e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.609375,
|
|
"reward_std": 0.27598243951797485,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 687,
|
|
"train_speed(iter/s)": 0.011053
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 796.0,
|
|
"completions/mean_length": 317.859375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.3989562191939693,
|
|
"grad_norm": 0.5630347583399415,
|
|
"kl": 0.0557861328125,
|
|
"learning_rate": 4.669764506151763e-07,
|
|
"loss": 5.585495091509074e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.3131476938724518,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 688,
|
|
"train_speed(iter/s)": 0.011063
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 758.0,
|
|
"completions/mean_length": 332.8515625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.3995360974195419,
|
|
"grad_norm": 0.47713188496364956,
|
|
"kl": 0.0386962890625,
|
|
"learning_rate": 4.6686612285575136e-07,
|
|
"loss": 3.859875141642988e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.16647969186306,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 689,
|
|
"train_speed(iter/s)": 0.011074
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 632.0,
|
|
"completions/mean_length": 298.0078125,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.40011597564511453,
|
|
"grad_norm": 0.5222194777133122,
|
|
"kl": 0.0364990234375,
|
|
"learning_rate": 4.667556257540181e-07,
|
|
"loss": 3.6399469536263496e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.3076016902923584,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 690,
|
|
"train_speed(iter/s)": 0.011085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 614.0,
|
|
"completions/mean_length": 303.484375,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.4006958538706872,
|
|
"grad_norm": 0.5364257520885441,
|
|
"kl": 0.0408935546875,
|
|
"learning_rate": 4.666449594075019e-07,
|
|
"loss": 4.085749969817698e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2829555869102478,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 691,
|
|
"train_speed(iter/s)": 0.011096
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 830.0,
|
|
"completions/mean_length": 309.5703125,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.4012757320962598,
|
|
"grad_norm": 0.516213097785082,
|
|
"kl": 0.038818359375,
|
|
"learning_rate": 4.665341239138776e-07,
|
|
"loss": 3.8889338611625135e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2772098779678345,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 692,
|
|
"train_speed(iter/s)": 0.011106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 848.0,
|
|
"completions/mean_length": 327.4609375,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.4018556103218324,
|
|
"grad_norm": 0.43040835034929176,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 4.6642311937096946e-07,
|
|
"loss": 3.9630773244425654e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.2632945775985718,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 693,
|
|
"train_speed(iter/s)": 0.011116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 643.0,
|
|
"completions/mean_length": 259.8671875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.402435488547405,
|
|
"grad_norm": 0.4832347591714459,
|
|
"kl": 0.051025390625,
|
|
"learning_rate": 4.6631194587675063e-07,
|
|
"loss": 5.1057428208878264e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.24106568098068237,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 694,
|
|
"train_speed(iter/s)": 0.011127
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 664.0,
|
|
"completions/mean_length": 282.9765625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.40301536677297767,
|
|
"grad_norm": 0.6396503200965064,
|
|
"kl": 0.0472412109375,
|
|
"learning_rate": 4.6620060352934376e-07,
|
|
"loss": 4.725471080746502e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66015625,
|
|
"reward_std": 0.31882143020629883,
|
|
"rewards/CSTORM/mean": 0.37109375,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 695,
|
|
"train_speed(iter/s)": 0.011138
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 651.0,
|
|
"completions/mean_length": 288.578125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.4035952449985503,
|
|
"grad_norm": 0.5076201714595111,
|
|
"kl": 0.0416259765625,
|
|
"learning_rate": 4.660890924270203e-07,
|
|
"loss": 4.16841430705972e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.2600978910923004,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 696,
|
|
"train_speed(iter/s)": 0.011148
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 878.0,
|
|
"completions/mean_length": 315.09375,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.4041751232241229,
|
|
"grad_norm": 0.5906399937432774,
|
|
"kl": 0.045654296875,
|
|
"learning_rate": 4.659774126682006e-07,
|
|
"loss": 4.563466427498497e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.26238325238227844,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 697,
|
|
"train_speed(iter/s)": 0.011156
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 976.0,
|
|
"completions/mean_length": 299.0703125,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.40475500144969556,
|
|
"grad_norm": 0.4981890333545344,
|
|
"kl": 0.1224365234375,
|
|
"learning_rate": 4.65865564351454e-07,
|
|
"loss": 0.00012262524978723377,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.30507156252861023,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 698,
|
|
"train_speed(iter/s)": 0.011165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 849.0,
|
|
"completions/mean_length": 307.8984375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.4053348796752682,
|
|
"grad_norm": 0.517559837292381,
|
|
"kl": 0.041748046875,
|
|
"learning_rate": 4.657535475754987e-07,
|
|
"loss": 4.182710836175829e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66796875,
|
|
"reward_std": 0.213870570063591,
|
|
"rewards/CSTORM/mean": 0.37109375,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 699,
|
|
"train_speed(iter/s)": 0.011176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 571.0,
|
|
"completions/mean_length": 263.0625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.4059147579008408,
|
|
"grad_norm": 0.47893239923862424,
|
|
"kl": 0.044921875,
|
|
"learning_rate": 4.6564136243920124e-07,
|
|
"loss": 4.5010732719674706e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.22103539109230042,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 700,
|
|
"train_speed(iter/s)": 0.011187
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 833.0,
|
|
"completions/mean_length": 330.390625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.40649463612641346,
|
|
"grad_norm": 0.42413614986722603,
|
|
"kl": 0.0362548828125,
|
|
"learning_rate": 4.6552900904157705e-07,
|
|
"loss": 3.626787292887457e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.2378501147031784,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 701,
|
|
"train_speed(iter/s)": 0.011191
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 766.0,
|
|
"completions/mean_length": 285.140625,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.4070745143519861,
|
|
"grad_norm": 0.4125108840225687,
|
|
"kl": 0.046142578125,
|
|
"learning_rate": 4.6541648748178997e-07,
|
|
"loss": 4.6070585085544735e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.22925812005996704,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 702,
|
|
"train_speed(iter/s)": 0.011202
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 733.0,
|
|
"completions/mean_length": 317.2734375,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.4076543925775587,
|
|
"grad_norm": 0.491786447022832,
|
|
"kl": 0.0433349609375,
|
|
"learning_rate": 4.6530379785915227e-07,
|
|
"loss": 4.338666258263402e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.2517244815826416,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 703,
|
|
"train_speed(iter/s)": 0.011213
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 705.0,
|
|
"completions/mean_length": 309.6953125,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.40823427080313135,
|
|
"grad_norm": 0.5693554103405045,
|
|
"kl": 0.041748046875,
|
|
"learning_rate": 4.651909402731245e-07,
|
|
"loss": 4.176121001364663e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.2799963653087616,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 704,
|
|
"train_speed(iter/s)": 0.011222
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 600.0,
|
|
"completions/mean_length": 299.390625,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.408814149028704,
|
|
"grad_norm": 0.5108062146453095,
|
|
"kl": 0.0439453125,
|
|
"learning_rate": 4.650779148233157e-07,
|
|
"loss": 4.395202631712891e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2439119815826416,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 705,
|
|
"train_speed(iter/s)": 0.011232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 622.0,
|
|
"completions/mean_length": 268.8828125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.4093940272542766,
|
|
"grad_norm": 0.59913269753714,
|
|
"kl": 0.0489501953125,
|
|
"learning_rate": 4.6496472160948264e-07,
|
|
"loss": 4.898625775240362e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.3190400004386902,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 706,
|
|
"train_speed(iter/s)": 0.011243
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 598.0,
|
|
"completions/mean_length": 269.6328125,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.40997390547984924,
|
|
"grad_norm": 0.5378145168299965,
|
|
"kl": 0.0499267578125,
|
|
"learning_rate": 4.648513607315306e-07,
|
|
"loss": 4.984791303286329e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2378501147031784,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 707,
|
|
"train_speed(iter/s)": 0.011255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 653.0,
|
|
"completions/mean_length": 282.6171875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.41055378370542184,
|
|
"grad_norm": 0.5133596061200112,
|
|
"kl": 0.041748046875,
|
|
"learning_rate": 4.6473783228951246e-07,
|
|
"loss": 4.1780869651120156e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.24339202046394348,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 708,
|
|
"train_speed(iter/s)": 0.011265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1726.0,
|
|
"completions/mean_length": 289.8203125,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.4111336619309945,
|
|
"grad_norm": 0.6264265824753606,
|
|
"kl": 0.0482177734375,
|
|
"learning_rate": 4.6462413638362933e-07,
|
|
"loss": 4.822457776754163e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.35235679149627686,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 709,
|
|
"train_speed(iter/s)": 0.011271
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 584.0,
|
|
"completions/mean_length": 253.875,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.41171354015656714,
|
|
"grad_norm": 0.4997046396220803,
|
|
"kl": 0.0489501953125,
|
|
"learning_rate": 4.6451027311422997e-07,
|
|
"loss": 4.89263838971965e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.2416265904903412,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 710,
|
|
"train_speed(iter/s)": 0.011282
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 582.0,
|
|
"completions/mean_length": 253.6875,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.41229341838213973,
|
|
"grad_norm": 0.5165096089516321,
|
|
"kl": 0.047607421875,
|
|
"learning_rate": 4.643962425818107e-07,
|
|
"loss": 4.751975939143449e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.209036186337471,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 711,
|
|
"train_speed(iter/s)": 0.011292
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 601.0,
|
|
"completions/mean_length": 269.03125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.4128732966077124,
|
|
"grad_norm": 0.5699335164034816,
|
|
"kl": 0.0489501953125,
|
|
"learning_rate": 4.6428204488701574e-07,
|
|
"loss": 4.891521166427992e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6328125,
|
|
"reward_std": 0.20163390040397644,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 712,
|
|
"train_speed(iter/s)": 0.011303
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 876.0,
|
|
"completions/mean_length": 282.4296875,
|
|
"completions/min_length": 68.0,
|
|
"epoch": 0.41345317483328503,
|
|
"grad_norm": 0.537036202074274,
|
|
"kl": 0.0413818359375,
|
|
"learning_rate": 4.641676801306367e-07,
|
|
"loss": 4.136448114877567e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.30144578218460083,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 713,
|
|
"train_speed(iter/s)": 0.011311
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 861.0,
|
|
"completions/mean_length": 289.0703125,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.4140330530588576,
|
|
"grad_norm": 0.5605628174694629,
|
|
"kl": 0.0419921875,
|
|
"learning_rate": 4.640531484136127e-07,
|
|
"loss": 4.199465911369771e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.21591851115226746,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 714,
|
|
"train_speed(iter/s)": 0.01132
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 782.0,
|
|
"completions/mean_length": 279.2578125,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.4146129312844303,
|
|
"grad_norm": 0.6322906216261482,
|
|
"kl": 0.0494384765625,
|
|
"learning_rate": 4.6393844983703007e-07,
|
|
"loss": 4.938908386975527e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6328125,
|
|
"reward_std": 0.3495104908943176,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 715,
|
|
"train_speed(iter/s)": 0.01133
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 748.0,
|
|
"completions/mean_length": 279.1640625,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.4151928095100029,
|
|
"grad_norm": 0.46708842593211364,
|
|
"kl": 0.0460205078125,
|
|
"learning_rate": 4.638235845021227e-07,
|
|
"loss": 4.6152330469340086e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.2025640904903412,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 716,
|
|
"train_speed(iter/s)": 0.01134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1103.0,
|
|
"completions/mean_length": 295.1875,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.4157726877355755,
|
|
"grad_norm": 0.6292642547152427,
|
|
"kl": 0.0423583984375,
|
|
"learning_rate": 4.6370855251027134e-07,
|
|
"loss": 4.2343239329056814e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.3737463653087616,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 717,
|
|
"train_speed(iter/s)": 0.011348
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 849.0,
|
|
"completions/mean_length": 285.015625,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.41635256596114817,
|
|
"grad_norm": 0.5965516764196838,
|
|
"kl": 0.04443359375,
|
|
"learning_rate": 4.635933539630042e-07,
|
|
"loss": 4.444544538273476e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.3214760720729828,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 718,
|
|
"train_speed(iter/s)": 0.011358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 857.0,
|
|
"completions/mean_length": 281.953125,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.41693244418672076,
|
|
"grad_norm": 0.6137958853869807,
|
|
"kl": 0.0494384765625,
|
|
"learning_rate": 4.6347798896199604e-07,
|
|
"loss": 4.937412450090051e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.34085866808891296,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 719,
|
|
"train_speed(iter/s)": 0.011368
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 637.0,
|
|
"completions/mean_length": 294.203125,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.4175123224122934,
|
|
"grad_norm": 0.49050383178589874,
|
|
"kl": 0.054443359375,
|
|
"learning_rate": 4.633624576090689e-07,
|
|
"loss": 5.4371699661714956e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.26951032876968384,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 720,
|
|
"train_speed(iter/s)": 0.011379
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 831.0,
|
|
"completions/mean_length": 288.609375,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.41809220063786606,
|
|
"grad_norm": 0.6395598975310285,
|
|
"kl": 0.0458984375,
|
|
"learning_rate": 4.632467600061915e-07,
|
|
"loss": 4.584930866258219e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.29296875,
|
|
"reward_std": 0.3423456847667694,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 721,
|
|
"train_speed(iter/s)": 0.011389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 529.0,
|
|
"completions/mean_length": 248.9609375,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.41867207886343866,
|
|
"grad_norm": 0.4864245277326502,
|
|
"kl": 0.0511474609375,
|
|
"learning_rate": 4.631308962554793e-07,
|
|
"loss": 5.117178079672158e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.1640625,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 722,
|
|
"train_speed(iter/s)": 0.011399
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1014.0,
|
|
"completions/mean_length": 302.34375,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.4192519570890113,
|
|
"grad_norm": 0.6443517922960597,
|
|
"kl": 0.0467529296875,
|
|
"learning_rate": 4.630148664591944e-07,
|
|
"loss": 4.671599162975326e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.3287726640701294,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 723,
|
|
"train_speed(iter/s)": 0.011408
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 606.0,
|
|
"completions/mean_length": 288.2265625,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.41983183531458396,
|
|
"grad_norm": 0.495842320790473,
|
|
"kl": 0.0452880859375,
|
|
"learning_rate": 4.628986707197455e-07,
|
|
"loss": 4.5276512537384406e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.2223757952451706,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 724,
|
|
"train_speed(iter/s)": 0.011419
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 502.0,
|
|
"completions/mean_length": 255.1640625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.42041171354015655,
|
|
"grad_norm": 0.6507927805803778,
|
|
"kl": 0.048095703125,
|
|
"learning_rate": 4.627823091396877e-07,
|
|
"loss": 4.807140794582665e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.2835353910923004,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 725,
|
|
"train_speed(iter/s)": 0.011429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 631.0,
|
|
"completions/mean_length": 273.578125,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.4209915917657292,
|
|
"grad_norm": 0.4826671993534285,
|
|
"kl": 0.049560546875,
|
|
"learning_rate": 4.6266578182172243e-07,
|
|
"loss": 4.9568541726330295e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.65234375,
|
|
"reward_std": 0.24106568098068237,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 726,
|
|
"train_speed(iter/s)": 0.011439
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1276.0,
|
|
"completions/mean_length": 294.78125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.42157146999130185,
|
|
"grad_norm": 0.584666022818361,
|
|
"kl": 0.0609130859375,
|
|
"learning_rate": 4.6254908886869766e-07,
|
|
"loss": 6.10531133133918e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.29649800062179565,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 727,
|
|
"train_speed(iter/s)": 0.011447
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 750.0,
|
|
"completions/mean_length": 264.0625,
|
|
"completions/min_length": 16.0,
|
|
"epoch": 0.42215134821687444,
|
|
"grad_norm": 0.49332080423210256,
|
|
"kl": 0.0521240234375,
|
|
"learning_rate": 4.624322303836072e-07,
|
|
"loss": 5.209428127272986e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.23545178771018982,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 728,
|
|
"train_speed(iter/s)": 0.011457
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 794.0,
|
|
"completions/mean_length": 301.2578125,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.4227312264424471,
|
|
"grad_norm": 0.4699653230399489,
|
|
"kl": 0.044189453125,
|
|
"learning_rate": 4.6231520646959115e-07,
|
|
"loss": 4.426510349730961e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.26480042934417725,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 729,
|
|
"train_speed(iter/s)": 0.011467
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 759.0,
|
|
"completions/mean_length": 297.21875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.4233111046680197,
|
|
"grad_norm": 0.5843925072367292,
|
|
"kl": 0.0482177734375,
|
|
"learning_rate": 4.6219801722993567e-07,
|
|
"loss": 4.817340595764108e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.2534751296043396,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 730,
|
|
"train_speed(iter/s)": 0.011476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 543.0,
|
|
"completions/mean_length": 275.984375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.42389098289359234,
|
|
"grad_norm": 0.437085411011544,
|
|
"kl": 0.0472412109375,
|
|
"learning_rate": 4.620806627680727e-07,
|
|
"loss": 4.722747689811513e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.18452188372612,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 731,
|
|
"train_speed(iter/s)": 0.011487
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 976.0,
|
|
"completions/mean_length": 305.1171875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.424470861119165,
|
|
"grad_norm": 0.536182411805789,
|
|
"kl": 0.045654296875,
|
|
"learning_rate": 4.6196314318758e-07,
|
|
"loss": 4.564054324873723e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.3184111714363098,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 732,
|
|
"train_speed(iter/s)": 0.011483
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 746.0,
|
|
"completions/mean_length": 272.09375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.4250507393447376,
|
|
"grad_norm": 0.5534610063910744,
|
|
"kl": 0.0472412109375,
|
|
"learning_rate": 4.6184545859218123e-07,
|
|
"loss": 4.7252811782527715e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.257100909948349,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 733,
|
|
"train_speed(iter/s)": 0.011493
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1202.0,
|
|
"completions/mean_length": 307.421875,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.42563061757031023,
|
|
"grad_norm": 0.4809886737723914,
|
|
"kl": 0.049072265625,
|
|
"learning_rate": 4.6172760908574546e-07,
|
|
"loss": 4.900756903225556e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59765625,
|
|
"reward_std": 0.2626469135284424,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 734,
|
|
"train_speed(iter/s)": 0.011501
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1162.0,
|
|
"completions/mean_length": 311.9609375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.4262104957958829,
|
|
"grad_norm": 0.4995256083503413,
|
|
"kl": 0.049560546875,
|
|
"learning_rate": 4.6160959477228766e-07,
|
|
"loss": 4.955611802870408e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.2204744666814804,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 735,
|
|
"train_speed(iter/s)": 0.011498
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 635.0,
|
|
"completions/mean_length": 275.8671875,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.4267903740214555,
|
|
"grad_norm": 0.5855932565136168,
|
|
"kl": 0.0501708984375,
|
|
"learning_rate": 4.6149141575596784e-07,
|
|
"loss": 5.0138150982093066e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.27734375,
|
|
"reward_std": 0.23303458094596863,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 736,
|
|
"train_speed(iter/s)": 0.011508
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 708.0,
|
|
"completions/mean_length": 272.734375,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.4273702522470281,
|
|
"grad_norm": 0.4934558029679181,
|
|
"kl": 0.0489501953125,
|
|
"learning_rate": 4.6137307214109174e-07,
|
|
"loss": 4.898578117717989e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.22507141530513763,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 737,
|
|
"train_speed(iter/s)": 0.011518
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 728.0,
|
|
"completions/mean_length": 297.8515625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.4279501304726008,
|
|
"grad_norm": 0.5077823123901771,
|
|
"kl": 0.0423583984375,
|
|
"learning_rate": 4.6125456403211003e-07,
|
|
"loss": 4.239362169755623e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.2494390904903412,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 738,
|
|
"train_speed(iter/s)": 0.011528
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 778.0,
|
|
"completions/mean_length": 273.3828125,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.42853000869817337,
|
|
"grad_norm": 0.6294025808070342,
|
|
"kl": 0.0504150390625,
|
|
"learning_rate": 4.6113589153361893e-07,
|
|
"loss": 5.050615072832443e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.3196197748184204,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 739,
|
|
"train_speed(iter/s)": 0.011538
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 706.0,
|
|
"completions/mean_length": 297.90625,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.429109886923746,
|
|
"grad_norm": 0.4812581118148176,
|
|
"kl": 0.3043212890625,
|
|
"learning_rate": 4.6101705475035953e-07,
|
|
"loss": 0.0003030894440598786,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.19781647622585297,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 740,
|
|
"train_speed(iter/s)": 0.011548
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1262.0,
|
|
"completions/mean_length": 309.84375,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.42968976514931867,
|
|
"grad_norm": 0.47199352770376957,
|
|
"kl": 0.047607421875,
|
|
"learning_rate": 4.608980537872179e-07,
|
|
"loss": 4.7590343456249684e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.2025640904903412,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 741,
|
|
"train_speed(iter/s)": 0.011555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1285.0,
|
|
"completions/mean_length": 308.25,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.43026964337489126,
|
|
"grad_norm": 0.5444884480482411,
|
|
"kl": 0.0401611328125,
|
|
"learning_rate": 4.6077888874922506e-07,
|
|
"loss": 4.012703357147984e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.2723156809806824,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 742,
|
|
"train_speed(iter/s)": 0.011563
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 820.0,
|
|
"completions/mean_length": 300.21875,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.4308495216004639,
|
|
"grad_norm": 0.4661624054710782,
|
|
"kl": 0.087158203125,
|
|
"learning_rate": 4.6065955974155695e-07,
|
|
"loss": 8.715942385606468e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.27845582365989685,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 743,
|
|
"train_speed(iter/s)": 0.011572
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 599.0,
|
|
"completions/mean_length": 274.0234375,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.4314293998260365,
|
|
"grad_norm": 0.5332433398587049,
|
|
"kl": 0.0482177734375,
|
|
"learning_rate": 4.6054006686953405e-07,
|
|
"loss": 4.816085856873542e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.19056488573551178,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 744,
|
|
"train_speed(iter/s)": 0.011582
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 591.0,
|
|
"completions/mean_length": 282.5,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.43200927805160916,
|
|
"grad_norm": 0.4927033840148636,
|
|
"kl": 0.0482177734375,
|
|
"learning_rate": 4.6042041023862167e-07,
|
|
"loss": 4.824970892514102e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6953125,
|
|
"reward_std": 0.25916770100593567,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 745,
|
|
"train_speed(iter/s)": 0.011591
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 934.0,
|
|
"completions/mean_length": 320.1796875,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.4325891562771818,
|
|
"grad_norm": 0.4210466277147917,
|
|
"kl": 0.043701171875,
|
|
"learning_rate": 4.603005899544294e-07,
|
|
"loss": 4.3739983084378764e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.21875,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 746,
|
|
"train_speed(iter/s)": 0.0116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 844.0,
|
|
"completions/mean_length": 293.734375,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.4331690345027544,
|
|
"grad_norm": 0.5734256880367001,
|
|
"kl": 0.0970458984375,
|
|
"learning_rate": 4.601806061227115e-07,
|
|
"loss": 9.698617941467091e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.4055122435092926,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 747,
|
|
"train_speed(iter/s)": 0.011609
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 893.0,
|
|
"completions/mean_length": 279.296875,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.43374891272832705,
|
|
"grad_norm": 0.607971166004982,
|
|
"kl": 0.052978515625,
|
|
"learning_rate": 4.600604588493666e-07,
|
|
"loss": 5.301490455167368e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.29873126745224,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 748,
|
|
"train_speed(iter/s)": 0.011619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 692.0,
|
|
"completions/mean_length": 246.53125,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.4343287909538997,
|
|
"grad_norm": 0.7126607169886496,
|
|
"kl": 0.054931640625,
|
|
"learning_rate": 4.5994014824043727e-07,
|
|
"loss": 5.492960553965531e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.26721763610839844,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 749,
|
|
"train_speed(iter/s)": 0.01163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1288.0,
|
|
"completions/mean_length": 347.625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.4349086691794723,
|
|
"grad_norm": 0.4138866045029754,
|
|
"kl": 0.072265625,
|
|
"learning_rate": 4.598196744021106e-07,
|
|
"loss": 7.214403012767434e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.2650640904903412,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 750,
|
|
"train_speed(iter/s)": 0.011577
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 890.0,
|
|
"completions/mean_length": 264.140625,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.43548854740504495,
|
|
"grad_norm": 0.46582863487206966,
|
|
"kl": 0.0499267578125,
|
|
"learning_rate": 4.5969903744071746e-07,
|
|
"loss": 4.997780342819169e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.1484375,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 751,
|
|
"train_speed(iter/s)": 0.011586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 549.0,
|
|
"completions/mean_length": 276.890625,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.4360684256306176,
|
|
"grad_norm": 0.4256459082379957,
|
|
"kl": 0.0482177734375,
|
|
"learning_rate": 4.59578237462733e-07,
|
|
"loss": 4.8152698582271114e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.17778617143630981,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 752,
|
|
"train_speed(iter/s)": 0.011596
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1330.0,
|
|
"completions/mean_length": 302.515625,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.4366483038561902,
|
|
"grad_norm": 0.5617903252044147,
|
|
"kl": 0.1409912109375,
|
|
"learning_rate": 4.5945727457477594e-07,
|
|
"loss": 0.00014045665739104152,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.2576806843280792,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 753,
|
|
"train_speed(iter/s)": 0.011544
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 933.0,
|
|
"completions/mean_length": 306.8828125,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.43722818208176284,
|
|
"grad_norm": 0.540335013211198,
|
|
"kl": 0.0460205078125,
|
|
"learning_rate": 4.59336148883609e-07,
|
|
"loss": 4.6041837777011096e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.2829744815826416,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 754,
|
|
"train_speed(iter/s)": 0.011554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 860.0,
|
|
"completions/mean_length": 269.875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.43780806030733543,
|
|
"grad_norm": 0.5346940981351561,
|
|
"kl": 0.0517578125,
|
|
"learning_rate": 4.5921486049613837e-07,
|
|
"loss": 5.172981764189899e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59765625,
|
|
"reward_std": 0.20903617143630981,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 755,
|
|
"train_speed(iter/s)": 0.011563
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 745.0,
|
|
"completions/mean_length": 289.8671875,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.4383879385329081,
|
|
"grad_norm": 0.5865865449310969,
|
|
"kl": 0.04833984375,
|
|
"learning_rate": 4.5909340951941426e-07,
|
|
"loss": 4.832018748857081e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.2890174686908722,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 756,
|
|
"train_speed(iter/s)": 0.011571
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 739.0,
|
|
"completions/mean_length": 257.4453125,
|
|
"completions/min_length": 67.0,
|
|
"epoch": 0.43896781675848073,
|
|
"grad_norm": 0.597440417466308,
|
|
"kl": 0.04931640625,
|
|
"learning_rate": 4.589717960606299e-07,
|
|
"loss": 4.928756243316457e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2948041260242462,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 757,
|
|
"train_speed(iter/s)": 0.011581
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 833.0,
|
|
"completions/mean_length": 277.21875,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.4395476949840533,
|
|
"grad_norm": 0.46731921740781823,
|
|
"kl": 0.05078125,
|
|
"learning_rate": 4.5885002022712217e-07,
|
|
"loss": 5.083643918624148e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62109375,
|
|
"reward_std": 0.24283519387245178,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 758,
|
|
"train_speed(iter/s)": 0.01159
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 638.0,
|
|
"completions/mean_length": 290.2265625,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.440127573209626,
|
|
"grad_norm": 0.566656105949344,
|
|
"kl": 0.0433349609375,
|
|
"learning_rate": 4.587280821263712e-07,
|
|
"loss": 4.3331114284228534e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.25215357542037964,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 759,
|
|
"train_speed(iter/s)": 0.011585
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 696.0,
|
|
"completions/mean_length": 254.625,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.4407074514351986,
|
|
"grad_norm": 0.6275152889329593,
|
|
"kl": 0.0511474609375,
|
|
"learning_rate": 4.586059818660005e-07,
|
|
"loss": 5.121505819261074e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.3058510422706604,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 760,
|
|
"train_speed(iter/s)": 0.011596
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1009.0,
|
|
"completions/mean_length": 319.6484375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.4412873296607712,
|
|
"grad_norm": 0.6111190198265531,
|
|
"kl": 0.0455322265625,
|
|
"learning_rate": 4.5848371955377633e-07,
|
|
"loss": 4.561898822430521e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.3660656809806824,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 761,
|
|
"train_speed(iter/s)": 0.011604
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1199.0,
|
|
"completions/mean_length": 287.2109375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.44186720788634387,
|
|
"grad_norm": 0.4832894475474327,
|
|
"kl": 0.0484619140625,
|
|
"learning_rate": 4.5836129529760844e-07,
|
|
"loss": 4.8542391596129164e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 762,
|
|
"train_speed(iter/s)": 0.011609
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 920.0,
|
|
"completions/mean_length": 294.140625,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.4424470861119165,
|
|
"grad_norm": 0.5921506897721143,
|
|
"kl": 0.050537109375,
|
|
"learning_rate": 4.5823870920554915e-07,
|
|
"loss": 5.0518563512014225e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.3714609742164612,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 763,
|
|
"train_speed(iter/s)": 0.011618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 959.0,
|
|
"completions/mean_length": 283.6328125,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.4430269643374891,
|
|
"grad_norm": 0.584896403557625,
|
|
"kl": 0.0447998046875,
|
|
"learning_rate": 4.5811596138579385e-07,
|
|
"loss": 4.4835545850219205e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.3390023708343506,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 764,
|
|
"train_speed(iter/s)": 0.011626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 610.0,
|
|
"completions/mean_length": 273.25,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.44360684256306177,
|
|
"grad_norm": 0.42783786490645676,
|
|
"kl": 0.0828857421875,
|
|
"learning_rate": 4.5799305194668054e-07,
|
|
"loss": 8.277782762888819e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.1687650829553604,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 765,
|
|
"train_speed(iter/s)": 0.011636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1327.0,
|
|
"completions/mean_length": 339.234375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.44418672078863436,
|
|
"grad_norm": 0.5320922032778479,
|
|
"kl": 0.04296875,
|
|
"learning_rate": 4.5786998099668984e-07,
|
|
"loss": 4.3011506932089105e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.3419354557991028,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 766,
|
|
"train_speed(iter/s)": 0.011643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 846.0,
|
|
"completions/mean_length": 298.828125,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.444766599014207,
|
|
"grad_norm": 0.5012240970597046,
|
|
"kl": 0.04541015625,
|
|
"learning_rate": 4.5774674864444505e-07,
|
|
"loss": 4.545697083813138e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.20498128235340118,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 767,
|
|
"train_speed(iter/s)": 0.011653
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 752.0,
|
|
"completions/mean_length": 269.6796875,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.44534647723977966,
|
|
"grad_norm": 0.6515786840673636,
|
|
"kl": 0.0516357421875,
|
|
"learning_rate": 4.5762335499871186e-07,
|
|
"loss": 5.1627750508487225e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.3892395496368408,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 768,
|
|
"train_speed(iter/s)": 0.011663
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 945.0,
|
|
"completions/mean_length": 314.953125,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.44592635546535225,
|
|
"grad_norm": 0.44435889922396105,
|
|
"kl": 0.0489501953125,
|
|
"learning_rate": 4.574998001683983e-07,
|
|
"loss": 4.9033969844458625e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 769,
|
|
"train_speed(iter/s)": 0.011671
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1577.0,
|
|
"completions/mean_length": 280.8828125,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.4465062336909249,
|
|
"grad_norm": 0.3383400202667036,
|
|
"kl": 0.047119140625,
|
|
"learning_rate": 4.5737608426255466e-07,
|
|
"loss": 4.70992672489956e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.10881409049034119,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 770,
|
|
"train_speed(iter/s)": 0.011678
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 742.0,
|
|
"completions/mean_length": 302.9453125,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.44708611191649755,
|
|
"grad_norm": 0.5127897714959344,
|
|
"kl": 0.046875,
|
|
"learning_rate": 4.5725220739037346e-07,
|
|
"loss": 4.685645762947388e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.286600261926651,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 771,
|
|
"train_speed(iter/s)": 0.011687
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 813.0,
|
|
"completions/mean_length": 263.65625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.44766599014207015,
|
|
"grad_norm": 0.5038147446603325,
|
|
"kl": 0.05859375,
|
|
"learning_rate": 4.5712816966118925e-07,
|
|
"loss": 5.8590288972482085e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.29296875,
|
|
"reward_std": 0.16712738573551178,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 772,
|
|
"train_speed(iter/s)": 0.011696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 683.0,
|
|
"completions/mean_length": 304.7734375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.4482458683676428,
|
|
"grad_norm": 0.5338006952327992,
|
|
"kl": 0.0460205078125,
|
|
"learning_rate": 4.570039711844785e-07,
|
|
"loss": 4.603011620929465e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.2855234742164612,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 773,
|
|
"train_speed(iter/s)": 0.011705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 806.0,
|
|
"completions/mean_length": 299.140625,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.44882574659321545,
|
|
"grad_norm": 0.49371665682224486,
|
|
"kl": 0.044189453125,
|
|
"learning_rate": 4.5687961206985965e-07,
|
|
"loss": 4.4107815483585e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.23743988573551178,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 774,
|
|
"train_speed(iter/s)": 0.011714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 533.0,
|
|
"completions/mean_length": 257.8671875,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.44940562481878804,
|
|
"grad_norm": 0.7195894081382942,
|
|
"kl": 0.06298828125,
|
|
"learning_rate": 4.5675509242709286e-07,
|
|
"loss": 6.299342931015417e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.3586822748184204,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 775,
|
|
"train_speed(iter/s)": 0.011724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 880.0,
|
|
"completions/mean_length": 299.1015625,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.4499855030443607,
|
|
"grad_norm": 0.5689677772376686,
|
|
"kl": 0.0667724609375,
|
|
"learning_rate": 4.5663041236608e-07,
|
|
"loss": 6.675611075479537e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.2841830551624298,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 776,
|
|
"train_speed(iter/s)": 0.011734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1071.0,
|
|
"completions/mean_length": 287.3046875,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.45056538126993334,
|
|
"grad_norm": 0.47731331354513856,
|
|
"kl": 0.0484619140625,
|
|
"learning_rate": 4.565055719968645e-07,
|
|
"loss": 4.845514558837749e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.29296875,
|
|
"reward_std": 0.19099397957324982,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 777,
|
|
"train_speed(iter/s)": 0.01174
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 526.0,
|
|
"completions/mean_length": 262.1171875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.45114525949550593,
|
|
"grad_norm": 0.5506705321549186,
|
|
"kl": 0.0526123046875,
|
|
"learning_rate": 4.563805714296313e-07,
|
|
"loss": 5.2568291721399873e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6640625,
|
|
"reward_std": 0.2645031809806824,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 778,
|
|
"train_speed(iter/s)": 0.01175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 635.0,
|
|
"completions/mean_length": 257.4296875,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.4517251377210786,
|
|
"grad_norm": 0.5109313169316224,
|
|
"kl": 0.0487060546875,
|
|
"learning_rate": 4.562554107747067e-07,
|
|
"loss": 4.868873656960204e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.3077523708343506,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 779,
|
|
"train_speed(iter/s)": 0.01176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1082.0,
|
|
"completions/mean_length": 294.5546875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.4523050159466512,
|
|
"grad_norm": 0.5706353442098442,
|
|
"kl": 0.05078125,
|
|
"learning_rate": 4.561300901425584e-07,
|
|
"loss": 5.069937469670549e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.29881805181503296,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 780,
|
|
"train_speed(iter/s)": 0.011768
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1138.0,
|
|
"completions/mean_length": 289.8046875,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.45288489417222383,
|
|
"grad_norm": 0.5906237310706888,
|
|
"kl": 0.048828125,
|
|
"learning_rate": 4.5600460964379506e-07,
|
|
"loss": 4.889685078524053e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.3154330849647522,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 781,
|
|
"train_speed(iter/s)": 0.011776
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1267.0,
|
|
"completions/mean_length": 259.5390625,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.4534647723977965,
|
|
"grad_norm": 497.00872298425867,
|
|
"kl": 532.0264892578125,
|
|
"learning_rate": 4.558789693891668e-07,
|
|
"loss": 0.5313054919242859,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.3346838653087616,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 782,
|
|
"train_speed(iter/s)": 0.011711
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1018.0,
|
|
"completions/mean_length": 276.1015625,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.4540446506233691,
|
|
"grad_norm": 0.6845851959739953,
|
|
"kl": 0.0584716796875,
|
|
"learning_rate": 4.5575316948956445e-07,
|
|
"loss": 5.837050048285164e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.3317508101463318,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 783,
|
|
"train_speed(iter/s)": 0.01172
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 770.0,
|
|
"completions/mean_length": 272.8828125,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.4546245288489417,
|
|
"grad_norm": 0.55854103644927,
|
|
"kl": 0.051513671875,
|
|
"learning_rate": 4.556272100560198e-07,
|
|
"loss": 5.153549136593938e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.28580188751220703,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 784,
|
|
"train_speed(iter/s)": 0.01173
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 309.0703125,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.45520440707451437,
|
|
"grad_norm": 0.6622492769538317,
|
|
"kl": 0.0460205078125,
|
|
"learning_rate": 4.5550109119970555e-07,
|
|
"loss": 4.598196755978279e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.38248497247695923,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 785,
|
|
"train_speed(iter/s)": 0.011665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1189.0,
|
|
"completions/mean_length": 320.421875,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.45578428530008697,
|
|
"grad_norm": 0.5725725157723329,
|
|
"kl": 0.0439453125,
|
|
"learning_rate": 4.5537481303193486e-07,
|
|
"loss": 4.3964842916466296e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.3504406809806824,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 786,
|
|
"train_speed(iter/s)": 0.011671
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 543.0,
|
|
"completions/mean_length": 269.3046875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.4563641635256596,
|
|
"grad_norm": 0.5289283447142948,
|
|
"kl": 0.052734375,
|
|
"learning_rate": 4.5524837566416196e-07,
|
|
"loss": 5.27412339579314e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.2713854908943176,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 787,
|
|
"train_speed(iter/s)": 0.011681
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 607.0,
|
|
"completions/mean_length": 266.2421875,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.45694404175123227,
|
|
"grad_norm": 0.5653938733327638,
|
|
"kl": 0.05029296875,
|
|
"learning_rate": 4.5512177920798106e-07,
|
|
"loss": 5.0235972594236955e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.2103765904903412,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 788,
|
|
"train_speed(iter/s)": 0.011691
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 824.0,
|
|
"completions/mean_length": 308.2109375,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.45752391997680486,
|
|
"grad_norm": 0.5437804889431076,
|
|
"kl": 0.047607421875,
|
|
"learning_rate": 4.5499502377512714e-07,
|
|
"loss": 4.7593457566108555e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.31622734665870667,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 789,
|
|
"train_speed(iter/s)": 0.0117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 338.046875,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.4581037982023775,
|
|
"grad_norm": 0.6416058106607835,
|
|
"kl": 0.052001953125,
|
|
"learning_rate": 4.548681094774754e-07,
|
|
"loss": 5.198130020289682e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.4962494373321533,
|
|
"rewards/CSTORM/mean": 0.38671875,
|
|
"rewards/CSTORM/std": 0.7187446355819702,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 790,
|
|
"train_speed(iter/s)": 0.011635
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1748.0,
|
|
"completions/mean_length": 296.546875,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.4586836764279501,
|
|
"grad_norm": 0.569797909337923,
|
|
"kl": 0.0489501953125,
|
|
"learning_rate": 4.547410364270412e-07,
|
|
"loss": 4.8904788854997605e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.24609375,
|
|
"reward_std": 0.2945445775985718,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 791,
|
|
"train_speed(iter/s)": 0.01164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 688.0,
|
|
"completions/mean_length": 283.6171875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.45926355465352275,
|
|
"grad_norm": 0.5702447758504531,
|
|
"kl": 0.0533447265625,
|
|
"learning_rate": 4.5461380473598005e-07,
|
|
"loss": 5.333546505426057e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.2775791585445404,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 792,
|
|
"train_speed(iter/s)": 0.011649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 898.0,
|
|
"completions/mean_length": 284.2109375,
|
|
"completions/min_length": 67.0,
|
|
"epoch": 0.4598434328790954,
|
|
"grad_norm": 0.5169951948909898,
|
|
"kl": 0.0576171875,
|
|
"learning_rate": 4.5448641451658756e-07,
|
|
"loss": 5.762156069977209e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.25250399112701416,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 793,
|
|
"train_speed(iter/s)": 0.011658
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 531.0,
|
|
"completions/mean_length": 252.2734375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.460423311104668,
|
|
"grad_norm": 0.5853344938556535,
|
|
"kl": 0.05224609375,
|
|
"learning_rate": 4.54358865881299e-07,
|
|
"loss": 5.22563059348613e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.23083597421646118,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 794,
|
|
"train_speed(iter/s)": 0.011667
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 892.0,
|
|
"completions/mean_length": 290.1640625,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.46100318933024065,
|
|
"grad_norm": 0.536644140431628,
|
|
"kl": 0.048095703125,
|
|
"learning_rate": 4.5423115894268984e-07,
|
|
"loss": 4.811865437659435e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.203125,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 795,
|
|
"train_speed(iter/s)": 0.011676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 635.0,
|
|
"completions/mean_length": 254.3046875,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.4615830675558133,
|
|
"grad_norm": 0.5366963091417153,
|
|
"kl": 0.056640625,
|
|
"learning_rate": 4.5410329381347483e-07,
|
|
"loss": 5.664518539560959e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.24809867143630981,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 796,
|
|
"train_speed(iter/s)": 0.011686
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1007.0,
|
|
"completions/mean_length": 280.046875,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.4621629457813859,
|
|
"grad_norm": 0.6651094897540375,
|
|
"kl": 0.0518798828125,
|
|
"learning_rate": 4.539752706065088e-07,
|
|
"loss": 5.200096347834915e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.3971838653087616,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 797,
|
|
"train_speed(iter/s)": 0.011695
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 987.0,
|
|
"completions/mean_length": 280.9765625,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.46274282400695854,
|
|
"grad_norm": 0.6067558026986666,
|
|
"kl": 0.053466796875,
|
|
"learning_rate": 4.5384708943478586e-07,
|
|
"loss": 5.3458657930605114e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.30131393671035767,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 798,
|
|
"train_speed(iter/s)": 0.011703
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 704.0,
|
|
"completions/mean_length": 263.4921875,
|
|
"completions/min_length": 68.0,
|
|
"epoch": 0.4633227022325312,
|
|
"grad_norm": 0.5767115639531039,
|
|
"kl": 0.05615234375,
|
|
"learning_rate": 4.537187504114395e-07,
|
|
"loss": 5.6206059525720775e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.20178458094596863,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 799,
|
|
"train_speed(iter/s)": 0.011712
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 965.0,
|
|
"completions/mean_length": 285.0234375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.4639025804581038,
|
|
"grad_norm": 0.6151446811876212,
|
|
"kl": 0.0517578125,
|
|
"learning_rate": 4.535902536497426e-07,
|
|
"loss": 5.18416563863866e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.30869734287261963,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 800,
|
|
"train_speed(iter/s)": 0.01172
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 612.0,
|
|
"completions/mean_length": 268.375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.46448245868367644,
|
|
"grad_norm": 0.5105866031820656,
|
|
"kl": 0.0533447265625,
|
|
"learning_rate": 4.534615992631072e-07,
|
|
"loss": 5.339584095054306e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.19759789109230042,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 801,
|
|
"train_speed(iter/s)": 0.011725
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1640.0,
|
|
"completions/mean_length": 283.0390625,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.46506233690924903,
|
|
"grad_norm": 0.5676698224459538,
|
|
"kl": 0.056396484375,
|
|
"learning_rate": 4.5333278736508454e-07,
|
|
"loss": 5.65079171792604e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.303433895111084,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 802,
|
|
"train_speed(iter/s)": 0.01173
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 941.0,
|
|
"completions/mean_length": 267.0859375,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.4656422151348217,
|
|
"grad_norm": 0.616556623383548,
|
|
"kl": 0.0589599609375,
|
|
"learning_rate": 4.532038180693649e-07,
|
|
"loss": 5.898288145544939e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.3275640904903412,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 803,
|
|
"train_speed(iter/s)": 0.011739
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1044.0,
|
|
"completions/mean_length": 304.046875,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.46622209336039433,
|
|
"grad_norm": 0.6020791599790135,
|
|
"kl": 0.0545654296875,
|
|
"learning_rate": 4.5307469148977747e-07,
|
|
"loss": 5.4494928917847574e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.234375,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 804,
|
|
"train_speed(iter/s)": 0.011747
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 794.0,
|
|
"completions/mean_length": 243.7578125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.4668019715859669,
|
|
"grad_norm": 0.5263395245861361,
|
|
"kl": 0.064208984375,
|
|
"learning_rate": 4.5294540774029007e-07,
|
|
"loss": 6.429101631511003e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.19056488573551178,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 805,
|
|
"train_speed(iter/s)": 0.011757
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 880.0,
|
|
"completions/mean_length": 270.7109375,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.4673818498115396,
|
|
"grad_norm": 0.658547587032844,
|
|
"kl": 0.056884765625,
|
|
"learning_rate": 4.528159669350095e-07,
|
|
"loss": 5.685858923243359e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.28394562005996704,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 806,
|
|
"train_speed(iter/s)": 0.011766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 569.0,
|
|
"completions/mean_length": 270.3515625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.4679617280371122,
|
|
"grad_norm": 0.5963507125025885,
|
|
"kl": 0.0576171875,
|
|
"learning_rate": 4.5268636918818104e-07,
|
|
"loss": 5.7645786000648513e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.2925376296043396,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 807,
|
|
"train_speed(iter/s)": 0.011762
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 535.0,
|
|
"completions/mean_length": 246.234375,
|
|
"completions/min_length": 63.0,
|
|
"epoch": 0.4685416062626848,
|
|
"grad_norm": 0.5505694418208115,
|
|
"kl": 0.057861328125,
|
|
"learning_rate": 4.5255661461418854e-07,
|
|
"loss": 5.7877201470546424e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.1909751147031784,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 808,
|
|
"train_speed(iter/s)": 0.011771
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1094.0,
|
|
"completions/mean_length": 287.3125,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.46912148448825747,
|
|
"grad_norm": 0.6425035211489948,
|
|
"kl": 0.0615234375,
|
|
"learning_rate": 4.524267033275543e-07,
|
|
"loss": 6.155608571134508e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3359375,
|
|
"reward_std": 0.3172025680541992,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 809,
|
|
"train_speed(iter/s)": 0.011779
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 674.0,
|
|
"completions/mean_length": 285.6328125,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.4697013627138301,
|
|
"grad_norm": 0.4577241870863807,
|
|
"kl": 0.0555419921875,
|
|
"learning_rate": 4.522966354429387e-07,
|
|
"loss": 5.558703560382128e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.18978539109230042,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 810,
|
|
"train_speed(iter/s)": 0.011787
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1030.0,
|
|
"completions/mean_length": 285.484375,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.4702812409394027,
|
|
"grad_norm": 0.6463271180676909,
|
|
"kl": 0.046875,
|
|
"learning_rate": 4.521664110751406e-07,
|
|
"loss": 4.6836212277412415e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.4373231530189514,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 811,
|
|
"train_speed(iter/s)": 0.011795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 540.0,
|
|
"completions/mean_length": 261.1875,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.47086111916497536,
|
|
"grad_norm": 0.48007517836253694,
|
|
"kl": 0.05712890625,
|
|
"learning_rate": 4.5203603033909706e-07,
|
|
"loss": 5.711282210540958e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.1635015904903412,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 812,
|
|
"train_speed(iter/s)": 0.011805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 953.0,
|
|
"completions/mean_length": 254.1328125,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.471440997390548,
|
|
"grad_norm": 0.6104227361655374,
|
|
"kl": 0.057861328125,
|
|
"learning_rate": 4.519054933498827e-07,
|
|
"loss": 5.783262531622313e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.3151862323284149,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 813,
|
|
"train_speed(iter/s)": 0.011813
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 648.0,
|
|
"completions/mean_length": 255.265625,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.4720208756161206,
|
|
"grad_norm": 0.6000820183435552,
|
|
"kl": 0.0596923828125,
|
|
"learning_rate": 4.517748002227106e-07,
|
|
"loss": 5.970546408207156e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.2572515904903412,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 814,
|
|
"train_speed(iter/s)": 0.011823
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 640.0,
|
|
"completions/mean_length": 291.7421875,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.47260075384169326,
|
|
"grad_norm": 0.6449916314736883,
|
|
"kl": 0.05517578125,
|
|
"learning_rate": 4.5164395107293115e-07,
|
|
"loss": 5.5075433920137584e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.36757156252861023,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 815,
|
|
"train_speed(iter/s)": 0.011832
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 732.0,
|
|
"completions/mean_length": 268.4921875,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.47318063206726585,
|
|
"grad_norm": 0.5813820753098955,
|
|
"kl": 0.0582275390625,
|
|
"learning_rate": 4.5151294601603274e-07,
|
|
"loss": 5.824350955663249e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.2536258101463318,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 816,
|
|
"train_speed(iter/s)": 0.011841
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 642.0,
|
|
"completions/mean_length": 277.328125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.4737605102928385,
|
|
"grad_norm": 0.4502679078430543,
|
|
"kl": 0.052001953125,
|
|
"learning_rate": 4.5138178516764135e-07,
|
|
"loss": 5.203265754971653e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.2282869815826416,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 817,
|
|
"train_speed(iter/s)": 0.011849
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 759.0,
|
|
"completions/mean_length": 276.5,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.47434038851841115,
|
|
"grad_norm": 0.6300166772409733,
|
|
"kl": 0.05224609375,
|
|
"learning_rate": 4.5125046864352044e-07,
|
|
"loss": 5.228989903116599e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3203125,
|
|
"reward_std": 0.30198779702186584,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 818,
|
|
"train_speed(iter/s)": 0.011859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 701.0,
|
|
"completions/mean_length": 271.0,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.47492026674398374,
|
|
"grad_norm": 0.4317888835212311,
|
|
"kl": 0.0511474609375,
|
|
"learning_rate": 4.5111899655957076e-07,
|
|
"loss": 5.117311957292259e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.65234375,
|
|
"reward_std": 0.2067507952451706,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 819,
|
|
"train_speed(iter/s)": 0.011868
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1279.0,
|
|
"completions/mean_length": 304.796875,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.4755001449695564,
|
|
"grad_norm": 0.54244588595907,
|
|
"kl": 0.0494384765625,
|
|
"learning_rate": 4.509873690318304e-07,
|
|
"loss": 4.951481969328597e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.2626469135284424,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 820,
|
|
"train_speed(iter/s)": 0.011861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 742.0,
|
|
"completions/mean_length": 279.984375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.47608002319512904,
|
|
"grad_norm": 0.46542634913691205,
|
|
"kl": 0.0499267578125,
|
|
"learning_rate": 4.5085558617647485e-07,
|
|
"loss": 4.989586886949837e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.25038406252861023,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 821,
|
|
"train_speed(iter/s)": 0.01187
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 711.0,
|
|
"completions/mean_length": 283.2265625,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.47665990142070164,
|
|
"grad_norm": 0.6448034417707388,
|
|
"kl": 0.05126953125,
|
|
"learning_rate": 4.507236481098164e-07,
|
|
"loss": 5.1233728299848735e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.609375,
|
|
"reward_std": 0.3558359742164612,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 822,
|
|
"train_speed(iter/s)": 0.011878
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1148.0,
|
|
"completions/mean_length": 297.15625,
|
|
"completions/min_length": 117.0,
|
|
"epoch": 0.4772397796462743,
|
|
"grad_norm": 0.42323855416894174,
|
|
"kl": 0.048828125,
|
|
"learning_rate": 4.505915549483045e-07,
|
|
"loss": 4.877450555795804e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.2048494666814804,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 823,
|
|
"train_speed(iter/s)": 0.011883
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 926.0,
|
|
"completions/mean_length": 282.875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.47781965787184694,
|
|
"grad_norm": 0.48651914760288945,
|
|
"kl": 0.052734375,
|
|
"learning_rate": 4.5045930680852545e-07,
|
|
"loss": 5.267251253826544e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.24609375,
|
|
"reward_std": 0.19341117143630981,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 824,
|
|
"train_speed(iter/s)": 0.011891
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 648.0,
|
|
"completions/mean_length": 255.890625,
|
|
"completions/min_length": 71.0,
|
|
"epoch": 0.47839953609741953,
|
|
"grad_norm": 0.5243121936003101,
|
|
"kl": 0.0615234375,
|
|
"learning_rate": 4.503269038072024e-07,
|
|
"loss": 6.151093839434907e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.2067507952451706,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 825,
|
|
"train_speed(iter/s)": 0.011901
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 693.0,
|
|
"completions/mean_length": 278.8359375,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.4789794143229922,
|
|
"grad_norm": 0.48259354979915237,
|
|
"kl": 0.0491943359375,
|
|
"learning_rate": 4.501943460611949e-07,
|
|
"loss": 4.914707824354991e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.31862977147102356,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 826,
|
|
"train_speed(iter/s)": 0.01191
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 769.0,
|
|
"completions/mean_length": 281.4921875,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.4795592925485648,
|
|
"grad_norm": 0.42423018640449867,
|
|
"kl": 0.0552978515625,
|
|
"learning_rate": 4.500616336874995e-07,
|
|
"loss": 5.5360011174343526e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.54296875,
|
|
"reward_std": 0.2024322748184204,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 827,
|
|
"train_speed(iter/s)": 0.011919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 903.0,
|
|
"completions/mean_length": 240.71875,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.4801391707741374,
|
|
"grad_norm": 0.6596974072321903,
|
|
"kl": 0.06494140625,
|
|
"learning_rate": 4.4992876680324894e-07,
|
|
"loss": 6.486412166850641e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.2703275680541992,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 828,
|
|
"train_speed(iter/s)": 0.011926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 593.0,
|
|
"completions/mean_length": 265.828125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.4807190489997101,
|
|
"grad_norm": 0.557139237385628,
|
|
"kl": 0.0562744140625,
|
|
"learning_rate": 4.4979574552571236e-07,
|
|
"loss": 5.629271981888451e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.26775968074798584,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 829,
|
|
"train_speed(iter/s)": 0.011935
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 498.0,
|
|
"completions/mean_length": 231.25,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.48129892722528267,
|
|
"grad_norm": 0.4945674006240571,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.4966256997229543e-07,
|
|
"loss": 6.934891280252486e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.76171875,
|
|
"reward_std": 0.16257140040397644,
|
|
"rewards/CSTORM/mean": 0.40234375,
|
|
"rewards/CSTORM/std": 0.19899940490722656,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.859375,
|
|
"rewards/VQAORM/std": 0.3490002751350403,
|
|
"step": 830,
|
|
"train_speed(iter/s)": 0.011945
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 813.0,
|
|
"completions/mean_length": 306.5390625,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.4818788054508553,
|
|
"grad_norm": 0.5833577175727799,
|
|
"kl": 0.0555419921875,
|
|
"learning_rate": 4.495292402605395e-07,
|
|
"loss": 5.5435153626604006e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.3545103669166565,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 831,
|
|
"train_speed(iter/s)": 0.011953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 722.0,
|
|
"completions/mean_length": 236.875,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.48245868367642797,
|
|
"grad_norm": 0.5731304935103085,
|
|
"kl": 0.065185546875,
|
|
"learning_rate": 4.4939575650812246e-07,
|
|
"loss": 6.494788976851851e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.19837738573551178,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 832,
|
|
"train_speed(iter/s)": 0.011963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 546.0,
|
|
"completions/mean_length": 254.984375,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.48303856190200056,
|
|
"grad_norm": 0.4834562277980228,
|
|
"kl": 0.0574951171875,
|
|
"learning_rate": 4.4926211883285774e-07,
|
|
"loss": 5.7397006457904354e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.24037295579910278,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 833,
|
|
"train_speed(iter/s)": 0.011972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 841.0,
|
|
"completions/mean_length": 298.5625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.4836184401275732,
|
|
"grad_norm": 0.5318251097879966,
|
|
"kl": 0.052490234375,
|
|
"learning_rate": 4.4912832735269506e-07,
|
|
"loss": 5.241556209512055e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.27921685576438904,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 834,
|
|
"train_speed(iter/s)": 0.01198
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 568.0,
|
|
"completions/mean_length": 276.609375,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.48419831835314586,
|
|
"grad_norm": 0.5275839748202396,
|
|
"kl": 0.0552978515625,
|
|
"learning_rate": 4.489943821857196e-07,
|
|
"loss": 5.52659148524981e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.609375,
|
|
"reward_std": 0.2102447748184204,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 835,
|
|
"train_speed(iter/s)": 0.01199
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 583.0,
|
|
"completions/mean_length": 256.4296875,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.48477819657871846,
|
|
"grad_norm": 0.387479763856981,
|
|
"kl": 0.0562744140625,
|
|
"learning_rate": 4.4886028345015225e-07,
|
|
"loss": 5.626495840260759e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.1875,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 836,
|
|
"train_speed(iter/s)": 0.012
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 972.0,
|
|
"completions/mean_length": 289.2890625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.4853580748042911,
|
|
"grad_norm": 0.4559710206564325,
|
|
"kl": 0.0543212890625,
|
|
"learning_rate": 4.487260312643493e-07,
|
|
"loss": 5.428726944955997e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.23247367143630981,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 837,
|
|
"train_speed(iter/s)": 0.012008
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 699.0,
|
|
"completions/mean_length": 240.6953125,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.48593795302986376,
|
|
"grad_norm": 0.5393812487958853,
|
|
"kl": 0.05322265625,
|
|
"learning_rate": 4.4859162574680284e-07,
|
|
"loss": 5.314421287039295e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.25250399112701416,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 838,
|
|
"train_speed(iter/s)": 0.012017
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 731.0,
|
|
"completions/mean_length": 278.96875,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.48651783125543635,
|
|
"grad_norm": 0.5720345257000763,
|
|
"kl": 0.05224609375,
|
|
"learning_rate": 4.484570670161398e-07,
|
|
"loss": 5.2244795369915664e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2890625,
|
|
"reward_std": 0.3255120813846588,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 839,
|
|
"train_speed(iter/s)": 0.012026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/mean_length": 284.953125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.487097709481009,
|
|
"grad_norm": 0.6908673465587211,
|
|
"kl": 0.0565185546875,
|
|
"learning_rate": 4.4832235519112287e-07,
|
|
"loss": 5.649677768815309e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.54296875,
|
|
"reward_std": 0.3310391902923584,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 840,
|
|
"train_speed(iter/s)": 0.012034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 515.0,
|
|
"completions/mean_length": 252.09375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.4876775877065816,
|
|
"grad_norm": 0.5260770464855544,
|
|
"kl": 0.0589599609375,
|
|
"learning_rate": 4.481874903906495e-07,
|
|
"loss": 5.888798114028759e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.2723156809806824,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 841,
|
|
"train_speed(iter/s)": 0.012044
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 841.0,
|
|
"completions/mean_length": 272.5859375,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.48825746593215424,
|
|
"grad_norm": 0.575497471321445,
|
|
"kl": 0.05859375,
|
|
"learning_rate": 4.4805247273375226e-07,
|
|
"loss": 5.857298674527556e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.2872929871082306,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 842,
|
|
"train_speed(iter/s)": 0.012053
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/mean_length": 293.2578125,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.4888373441577269,
|
|
"grad_norm": 0.6118705205972363,
|
|
"kl": 0.05126953125,
|
|
"learning_rate": 4.4791730233959855e-07,
|
|
"loss": 5.121039066580124e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.3470744490623474,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 843,
|
|
"train_speed(iter/s)": 0.012059
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 528.0,
|
|
"completions/mean_length": 261.078125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.4894172223832995,
|
|
"grad_norm": 0.5167127894640474,
|
|
"kl": 0.0546875,
|
|
"learning_rate": 4.477819793274907e-07,
|
|
"loss": 5.465990398079157e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.23531997203826904,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 844,
|
|
"train_speed(iter/s)": 0.012068
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1085.0,
|
|
"completions/mean_length": 260.296875,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.48999710060887214,
|
|
"grad_norm": 0.6725588767724449,
|
|
"kl": 0.052001953125,
|
|
"learning_rate": 4.4764650381686575e-07,
|
|
"loss": 5.207112553762272e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.21875,
|
|
"reward_std": 0.380350261926651,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 845,
|
|
"train_speed(iter/s)": 0.012076
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 612.0,
|
|
"completions/mean_length": 248.9609375,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.4905769788344448,
|
|
"grad_norm": 0.5287887170022382,
|
|
"kl": 0.0623779296875,
|
|
"learning_rate": 4.4751087592729526e-07,
|
|
"loss": 6.227154517546296e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.2521347105503082,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 846,
|
|
"train_speed(iter/s)": 0.012085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 659.0,
|
|
"completions/mean_length": 279.046875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.4911568570600174,
|
|
"grad_norm": 0.5181186608752982,
|
|
"kl": 0.056640625,
|
|
"learning_rate": 4.4737509577848524e-07,
|
|
"loss": 5.6645076256245375e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.2757228910923004,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 847,
|
|
"train_speed(iter/s)": 0.012094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 675.0,
|
|
"completions/mean_length": 282.671875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.49173673528559003,
|
|
"grad_norm": 0.5601554803240695,
|
|
"kl": 0.0528564453125,
|
|
"learning_rate": 4.4723916349027623e-07,
|
|
"loss": 5.285263614496216e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.29881805181503296,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 848,
|
|
"train_speed(iter/s)": 0.012103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 951.0,
|
|
"completions/mean_length": 292.875,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.4923166135111627,
|
|
"grad_norm": 0.5505292151960324,
|
|
"kl": 0.0452880859375,
|
|
"learning_rate": 4.471030791826429e-07,
|
|
"loss": 4.531202284852043e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.3231137692928314,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 849,
|
|
"train_speed(iter/s)": 0.012112
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 641.0,
|
|
"completions/mean_length": 265.0546875,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.4928964917367353,
|
|
"grad_norm": 0.5506551585684628,
|
|
"kl": 0.0953369140625,
|
|
"learning_rate": 4.4696684297569415e-07,
|
|
"loss": 9.483173198532313e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.2451017051935196,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 850,
|
|
"train_speed(iter/s)": 0.01212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 543.0,
|
|
"completions/mean_length": 242.7421875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.4934763699623079,
|
|
"grad_norm": 0.470951330862777,
|
|
"kl": 0.056640625,
|
|
"learning_rate": 4.46830454989673e-07,
|
|
"loss": 5.6579578085802495e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.1639118194580078,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 851,
|
|
"train_speed(iter/s)": 0.012129
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/mean_length": 288.859375,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.4940562481878805,
|
|
"grad_norm": 0.6236986020834833,
|
|
"kl": 0.0548095703125,
|
|
"learning_rate": 4.4669391534495644e-07,
|
|
"loss": 5.4851909226272255e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.38779348134994507,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 852,
|
|
"train_speed(iter/s)": 0.012136
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 594.0,
|
|
"completions/mean_length": 280.921875,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.49463612641345317,
|
|
"grad_norm": 0.5447885794483512,
|
|
"kl": 0.0498046875,
|
|
"learning_rate": 4.4655722416205523e-07,
|
|
"loss": 4.985913255950436e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.1947515904903412,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 853,
|
|
"train_speed(iter/s)": 0.012144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 774.0,
|
|
"completions/mean_length": 242.515625,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.4952160046390258,
|
|
"grad_norm": 0.38660264384417486,
|
|
"kl": 0.0587158203125,
|
|
"learning_rate": 4.4642038156161376e-07,
|
|
"loss": 5.871593384654261e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.61328125,
|
|
"reward_std": 0.0973757952451706,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 854,
|
|
"train_speed(iter/s)": 0.012153
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1031.0,
|
|
"completions/mean_length": 257.359375,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.4957958828645984,
|
|
"grad_norm": 0.589683983891913,
|
|
"kl": 0.0557861328125,
|
|
"learning_rate": 4.462833876644104e-07,
|
|
"loss": 5.571276415139437e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.2841641902923584,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 855,
|
|
"train_speed(iter/s)": 0.01216
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 836.0,
|
|
"completions/mean_length": 265.6328125,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.49637576109017106,
|
|
"grad_norm": 0.4156666472462885,
|
|
"kl": 0.058837890625,
|
|
"learning_rate": 4.4614624259135683e-07,
|
|
"loss": 5.879694799659774e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.19759787619113922,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 856,
|
|
"train_speed(iter/s)": 0.012168
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 960.0,
|
|
"completions/mean_length": 280.9296875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.4969556393157437,
|
|
"grad_norm": 0.5685024276767681,
|
|
"kl": 0.05712890625,
|
|
"learning_rate": 4.4600894646349817e-07,
|
|
"loss": 5.722267087548971e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.2855234742164612,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 857,
|
|
"train_speed(iter/s)": 0.012176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 747.0,
|
|
"completions/mean_length": 268.0625,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.4975355175413163,
|
|
"grad_norm": 0.4785715586347527,
|
|
"kl": 0.0718994140625,
|
|
"learning_rate": 4.4587149940201284e-07,
|
|
"loss": 7.212496711872518e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.1755007952451706,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 858,
|
|
"train_speed(iter/s)": 0.012185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1377.0,
|
|
"completions/mean_length": 266.421875,
|
|
"completions/min_length": 64.0,
|
|
"epoch": 0.49811539576688896,
|
|
"grad_norm": 0.48115913739442284,
|
|
"kl": 0.05224609375,
|
|
"learning_rate": 4.4573390152821263e-07,
|
|
"loss": 5.221056198934093e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.2607266902923584,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 859,
|
|
"train_speed(iter/s)": 0.012191
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 524.0,
|
|
"completions/mean_length": 242.171875,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.4986952739924616,
|
|
"grad_norm": 0.5552297898004275,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 4.4559615296354226e-07,
|
|
"loss": 6.639459752477705e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.22934488952159882,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 860,
|
|
"train_speed(iter/s)": 0.012201
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 666.0,
|
|
"completions/mean_length": 282.0859375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.4992751522180342,
|
|
"grad_norm": 0.42386937981607253,
|
|
"kl": 0.078125,
|
|
"learning_rate": 4.454582538295795e-07,
|
|
"loss": 7.795200508553535e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.1493992656469345,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 861,
|
|
"train_speed(iter/s)": 0.012196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 673.0,
|
|
"completions/mean_length": 272.21875,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.49985503044360685,
|
|
"grad_norm": 0.5389718320966016,
|
|
"kl": 0.0570068359375,
|
|
"learning_rate": 4.453202042480352e-07,
|
|
"loss": 5.6993427278939635e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.20661897957324982,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 862,
|
|
"train_speed(iter/s)": 0.012206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 663.0,
|
|
"completions/mean_length": 264.875,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.5004349086691795,
|
|
"grad_norm": 0.5703958851341564,
|
|
"kl": 0.0654296875,
|
|
"learning_rate": 4.4518200434075266e-07,
|
|
"loss": 6.523144111270085e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.2541416883468628,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 863,
|
|
"train_speed(iter/s)": 0.012215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 530.0,
|
|
"completions/mean_length": 254.0390625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.5010147868947521,
|
|
"grad_norm": 0.6071008671702024,
|
|
"kl": 0.0537109375,
|
|
"learning_rate": 4.450436542297081e-07,
|
|
"loss": 5.3713625675300136e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.2494390904903412,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 864,
|
|
"train_speed(iter/s)": 0.012224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 835.0,
|
|
"completions/mean_length": 310.8046875,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.5015946651203247,
|
|
"grad_norm": 0.4956308839076791,
|
|
"kl": 0.0528564453125,
|
|
"learning_rate": 4.449051540370102e-07,
|
|
"loss": 5.2842686272924766e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.2109375,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 865,
|
|
"train_speed(iter/s)": 0.012231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1088.0,
|
|
"completions/mean_length": 305.640625,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.5021745433458974,
|
|
"grad_norm": 0.6149876863184435,
|
|
"kl": 0.0496826171875,
|
|
"learning_rate": 4.4476650388490024e-07,
|
|
"loss": 4.971005182596855e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6640625,
|
|
"reward_std": 0.27045938372612,
|
|
"rewards/CSTORM/mean": 0.375,
|
|
"rewards/CSTORM/std": 0.2173570692539215,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 866,
|
|
"train_speed(iter/s)": 0.012238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1208.0,
|
|
"completions/mean_length": 284.96875,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.50275442157147,
|
|
"grad_norm": 0.5586644269143993,
|
|
"kl": 0.05517578125,
|
|
"learning_rate": 4.446277038957516e-07,
|
|
"loss": 5.5118052841862664e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.28219497203826904,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 867,
|
|
"train_speed(iter/s)": 0.012245
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 654.0,
|
|
"completions/mean_length": 255.1640625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.5033342997970426,
|
|
"grad_norm": 0.5791207746088162,
|
|
"kl": 0.060546875,
|
|
"learning_rate": 4.444887541920703e-07,
|
|
"loss": 6.06482062721625e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.24432221055030823,
|
|
"rewards/CSTORM/mean": 0.2421875,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 868,
|
|
"train_speed(iter/s)": 0.012253
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 788.0,
|
|
"completions/mean_length": 271.6484375,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.5039141780226153,
|
|
"grad_norm": 0.6223827538097361,
|
|
"kl": 0.056884765625,
|
|
"learning_rate": 4.4434965489649405e-07,
|
|
"loss": 5.6817661970853806e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.31258678436279297,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 869,
|
|
"train_speed(iter/s)": 0.012261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 999.0,
|
|
"completions/mean_length": 285.9140625,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.5044940562481879,
|
|
"grad_norm": 0.600158075554856,
|
|
"kl": 0.04931640625,
|
|
"learning_rate": 4.442104061317929e-07,
|
|
"loss": 4.9240417865803465e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.2932041883468628,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 870,
|
|
"train_speed(iter/s)": 0.012269
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 699.0,
|
|
"completions/mean_length": 243.6640625,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.5050739344737605,
|
|
"grad_norm": 0.4770864166745593,
|
|
"kl": 0.062255859375,
|
|
"learning_rate": 4.440710080208687e-07,
|
|
"loss": 6.227698759175837e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6328125,
|
|
"reward_std": 0.17402857542037964,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 871,
|
|
"train_speed(iter/s)": 0.012278
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1178.0,
|
|
"completions/mean_length": 276.4921875,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.5056538126993332,
|
|
"grad_norm": 0.6406644269882118,
|
|
"kl": 0.0550537109375,
|
|
"learning_rate": 4.439314606867552e-07,
|
|
"loss": 5.504430737346411e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.3528915047645569,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 872,
|
|
"train_speed(iter/s)": 0.012273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1058.0,
|
|
"completions/mean_length": 317.9609375,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.5062336909249058,
|
|
"grad_norm": 0.5365951569377579,
|
|
"kl": 0.0474853515625,
|
|
"learning_rate": 4.437917642526177e-07,
|
|
"loss": 4.7500201617367566e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.3155648708343506,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 873,
|
|
"train_speed(iter/s)": 0.01228
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 987.0,
|
|
"completions/mean_length": 262.8125,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.5068135691504784,
|
|
"grad_norm": 0.5779793296543235,
|
|
"kl": 0.0499267578125,
|
|
"learning_rate": 4.4365191884175334e-07,
|
|
"loss": 4.986163548892364e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.2801281809806824,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 874,
|
|
"train_speed(iter/s)": 0.012289
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 746.0,
|
|
"completions/mean_length": 276.015625,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.5073934473760511,
|
|
"grad_norm": 0.6019619081647759,
|
|
"kl": 0.0545654296875,
|
|
"learning_rate": 4.435119245775905e-07,
|
|
"loss": 5.453558696899563e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.27787643671035767,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 875,
|
|
"train_speed(iter/s)": 0.012297
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 642.0,
|
|
"completions/mean_length": 260.0,
|
|
"completions/min_length": 68.0,
|
|
"epoch": 0.5079733256016237,
|
|
"grad_norm": 0.601935024715738,
|
|
"kl": 0.0582275390625,
|
|
"learning_rate": 4.4337178158368906e-07,
|
|
"loss": 5.819427315145731e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.19921875,
|
|
"reward_std": 0.31208568811416626,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 876,
|
|
"train_speed(iter/s)": 0.012306
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 634.0,
|
|
"completions/mean_length": 248.65625,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.5085532038271963,
|
|
"grad_norm": 0.571663049255324,
|
|
"kl": 0.0606689453125,
|
|
"learning_rate": 4.432314899837402e-07,
|
|
"loss": 6.070208837627433e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.609375,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 877,
|
|
"train_speed(iter/s)": 0.012315
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 677.0,
|
|
"completions/mean_length": 258.359375,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.509133082052769,
|
|
"grad_norm": 0.5879639032429466,
|
|
"kl": 0.05712890625,
|
|
"learning_rate": 4.4309104990156617e-07,
|
|
"loss": 5.710461118724197e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.24525238573551178,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 878,
|
|
"train_speed(iter/s)": 0.012324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 630.0,
|
|
"completions/mean_length": 246.09375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.5097129602783416,
|
|
"grad_norm": 0.484182956732098,
|
|
"kl": 0.0604248046875,
|
|
"learning_rate": 4.4295046146112025e-07,
|
|
"loss": 6.049593139323406e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.70703125,
|
|
"reward_std": 0.16498860716819763,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.828125,
|
|
"rewards/VQAORM/std": 0.3787541687488556,
|
|
"step": 879,
|
|
"train_speed(iter/s)": 0.012333
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 920.0,
|
|
"completions/mean_length": 270.828125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.5102928385039142,
|
|
"grad_norm": 0.630280464620679,
|
|
"kl": 0.0797119140625,
|
|
"learning_rate": 4.428097247864868e-07,
|
|
"loss": 7.977697532624006e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.3338854908943176,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 880,
|
|
"train_speed(iter/s)": 0.012341
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 634.0,
|
|
"completions/mean_length": 270.40625,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.5108727167294868,
|
|
"grad_norm": 0.4957545556974647,
|
|
"kl": 0.0528564453125,
|
|
"learning_rate": 4.4266884000188106e-07,
|
|
"loss": 5.293712820275687e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.2126619666814804,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 881,
|
|
"train_speed(iter/s)": 0.012349
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 820.0,
|
|
"completions/mean_length": 271.53125,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.5114525949550595,
|
|
"grad_norm": 0.5558411085657143,
|
|
"kl": 0.0576171875,
|
|
"learning_rate": 4.4252780723164855e-07,
|
|
"loss": 5.7528013712726533e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.23545178771018982,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 882,
|
|
"train_speed(iter/s)": 0.012358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 949.0,
|
|
"completions/mean_length": 288.6640625,
|
|
"completions/min_length": 66.0,
|
|
"epoch": 0.512032473180632,
|
|
"grad_norm": 0.7084552246224833,
|
|
"kl": 0.0552978515625,
|
|
"learning_rate": 4.423866266002661e-07,
|
|
"loss": 5.5294880439760163e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.32905110716819763,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 883,
|
|
"train_speed(iter/s)": 0.012363
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 747.0,
|
|
"completions/mean_length": 246.34375,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.5126123514062046,
|
|
"grad_norm": 0.6417745440792005,
|
|
"kl": 0.064697265625,
|
|
"learning_rate": 4.422452982323404e-07,
|
|
"loss": 6.459976430051029e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6953125,
|
|
"reward_std": 0.20044416189193726,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 884,
|
|
"train_speed(iter/s)": 0.012371
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1493.0,
|
|
"completions/mean_length": 308.03125,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.5131922296317774,
|
|
"grad_norm": 0.44606537500425836,
|
|
"kl": 0.052978515625,
|
|
"learning_rate": 4.42103822252609e-07,
|
|
"loss": 5.2972820412833244e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.289578378200531,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 885,
|
|
"train_speed(iter/s)": 0.012376
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 689.0,
|
|
"completions/mean_length": 250.734375,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.51377210785735,
|
|
"grad_norm": 0.5515398719556552,
|
|
"kl": 0.0623779296875,
|
|
"learning_rate": 4.419621987859393e-07,
|
|
"loss": 6.227513222256675e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.63671875,
|
|
"reward_std": 0.18935628235340118,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 886,
|
|
"train_speed(iter/s)": 0.012384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1507.0,
|
|
"completions/mean_length": 300.171875,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.5143519860829225,
|
|
"grad_norm": 0.43624958628665417,
|
|
"kl": 0.0517578125,
|
|
"learning_rate": 4.418204279573293e-07,
|
|
"loss": 5.169669748283923e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.23113328218460083,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 887,
|
|
"train_speed(iter/s)": 0.012389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 757.0,
|
|
"completions/mean_length": 261.5625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.5149318643084952,
|
|
"grad_norm": 0.5492726931417296,
|
|
"kl": 0.066162109375,
|
|
"learning_rate": 4.416785098919068e-07,
|
|
"loss": 6.614598532905802e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.64453125,
|
|
"reward_std": 0.26775968074798584,
|
|
"rewards/CSTORM/mean": 0.37109375,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 888,
|
|
"train_speed(iter/s)": 0.012396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 564.0,
|
|
"completions/mean_length": 249.8125,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.5155117425340678,
|
|
"grad_norm": 0.5362514679950159,
|
|
"kl": 0.061279296875,
|
|
"learning_rate": 4.415364447149296e-07,
|
|
"loss": 6.125588697614148e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.27315500378608704,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 889,
|
|
"train_speed(iter/s)": 0.012405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1733.0,
|
|
"completions/mean_length": 264.09375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.5160916207596404,
|
|
"grad_norm": 0.5856909051468966,
|
|
"kl": 0.06298828125,
|
|
"learning_rate": 4.413942325517855e-07,
|
|
"loss": 6.308223237283528e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59765625,
|
|
"reward_std": 0.2963140904903412,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 890,
|
|
"train_speed(iter/s)": 0.012409
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 575.0,
|
|
"completions/mean_length": 247.640625,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.5166714989852131,
|
|
"grad_norm": 0.5741955868216432,
|
|
"kl": 0.153564453125,
|
|
"learning_rate": 4.4125187352799166e-07,
|
|
"loss": 0.00015380802506115288,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.19382140040397644,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 891,
|
|
"train_speed(iter/s)": 0.012357
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1025.0,
|
|
"completions/mean_length": 235.6015625,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.5172513772107857,
|
|
"grad_norm": 0.4989768377410597,
|
|
"kl": 0.0673828125,
|
|
"learning_rate": 4.4110936776919527e-07,
|
|
"loss": 6.72597816446796e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.21188247203826904,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 892,
|
|
"train_speed(iter/s)": 0.012365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 556.0,
|
|
"completions/mean_length": 267.09375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.5178312554363583,
|
|
"grad_norm": 0.5217181946176175,
|
|
"kl": 0.0511474609375,
|
|
"learning_rate": 4.409667154011728e-07,
|
|
"loss": 5.110178608447313e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.2723156809806824,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 893,
|
|
"train_speed(iter/s)": 0.012374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 770.0,
|
|
"completions/mean_length": 262.4765625,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.518411133661931,
|
|
"grad_norm": 0.7013014760706121,
|
|
"kl": 0.060791015625,
|
|
"learning_rate": 4.4082391654983017e-07,
|
|
"loss": 6.0746478993678465e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.3532869815826416,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 894,
|
|
"train_speed(iter/s)": 0.012382
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1017.0,
|
|
"completions/mean_length": 275.1953125,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.5189910118875036,
|
|
"grad_norm": 0.5479371253310473,
|
|
"kl": 0.061279296875,
|
|
"learning_rate": 4.4068097134120274e-07,
|
|
"loss": 6.129505345597863e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2109375,
|
|
"reward_std": 0.2980385422706604,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 895,
|
|
"train_speed(iter/s)": 0.01239
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1266.0,
|
|
"completions/mean_length": 299.1796875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.5195708901130762,
|
|
"grad_norm": 0.47566825072367863,
|
|
"kl": 0.05517578125,
|
|
"learning_rate": 4.4053787990145464e-07,
|
|
"loss": 5.5158794566523284e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.1556890904903412,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 896,
|
|
"train_speed(iter/s)": 0.012394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 683.0,
|
|
"completions/mean_length": 260.6015625,
|
|
"completions/min_length": 71.0,
|
|
"epoch": 0.5201507683386489,
|
|
"grad_norm": 0.38645774227202007,
|
|
"kl": 0.057373046875,
|
|
"learning_rate": 4.403946423568794e-07,
|
|
"loss": 5.735354352509603e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.1399322748184204,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 897,
|
|
"train_speed(iter/s)": 0.012403
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/mean_length": 275.0703125,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.5207306465642215,
|
|
"grad_norm": 0.5479924703972112,
|
|
"kl": 0.05859375,
|
|
"learning_rate": 4.402512588338996e-07,
|
|
"loss": 5.8708410506369546e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.26571178436279297,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 898,
|
|
"train_speed(iter/s)": 0.01241
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 555.0,
|
|
"completions/mean_length": 249.4921875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.5213105247897941,
|
|
"grad_norm": 0.510989928845423,
|
|
"kl": 0.0638427734375,
|
|
"learning_rate": 4.4010772945906614e-07,
|
|
"loss": 6.393828516593203e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.26279348134994507,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 899,
|
|
"train_speed(iter/s)": 0.012419
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 548.0,
|
|
"completions/mean_length": 255.109375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.5218904030153668,
|
|
"grad_norm": 0.480755226476582,
|
|
"kl": 0.0565185546875,
|
|
"learning_rate": 4.399640543590594e-07,
|
|
"loss": 5.6406090152449906e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.2144126147031784,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 900,
|
|
"train_speed(iter/s)": 0.012428
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1246.0,
|
|
"completions/mean_length": 270.0546875,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.5224702812409394,
|
|
"grad_norm": 0.49946878565119746,
|
|
"kl": 0.065185546875,
|
|
"learning_rate": 4.398202336606877e-07,
|
|
"loss": 6.51619047857821e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.21400238573551178,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 901,
|
|
"train_speed(iter/s)": 0.012429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 561.0,
|
|
"completions/mean_length": 251.46875,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.523050159466512,
|
|
"grad_norm": 0.6272121374088585,
|
|
"kl": 0.066162109375,
|
|
"learning_rate": 4.396762674908882e-07,
|
|
"loss": 6.63420360069722e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.30144575238227844,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 902,
|
|
"train_speed(iter/s)": 0.012438
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 818.0,
|
|
"completions/mean_length": 263.640625,
|
|
"completions/min_length": 67.0,
|
|
"epoch": 0.5236300376920847,
|
|
"grad_norm": 0.4957926131191077,
|
|
"kl": 0.0618896484375,
|
|
"learning_rate": 4.3953215597672645e-07,
|
|
"loss": 6.189158011693507e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.2494390904903412,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 903,
|
|
"train_speed(iter/s)": 0.012447
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 673.0,
|
|
"completions/mean_length": 261.6328125,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.5242099159176573,
|
|
"grad_norm": 0.7648726443400004,
|
|
"kl": 0.057373046875,
|
|
"learning_rate": 4.393878992453961e-07,
|
|
"loss": 5.732944919145666e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.234375,
|
|
"reward_std": 0.35164928436279297,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 904,
|
|
"train_speed(iter/s)": 0.012456
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 911.0,
|
|
"completions/mean_length": 285.9765625,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.5247897941432299,
|
|
"grad_norm": 0.5359597203140376,
|
|
"kl": 0.0611572265625,
|
|
"learning_rate": 4.392434974242193e-07,
|
|
"loss": 6.12898584222421e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.2885015904903412,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 905,
|
|
"train_speed(iter/s)": 0.012464
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 576.0,
|
|
"completions/mean_length": 255.0390625,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.5253696723688025,
|
|
"grad_norm": 0.6694888624798031,
|
|
"kl": 0.061767578125,
|
|
"learning_rate": 4.390989506406459e-07,
|
|
"loss": 6.17511905147694e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.29000747203826904,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 906,
|
|
"train_speed(iter/s)": 0.012473
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 628.0,
|
|
"completions/mean_length": 267.171875,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.5259495505943752,
|
|
"grad_norm": 0.5714499375279215,
|
|
"kl": 0.06103515625,
|
|
"learning_rate": 4.389542590222538e-07,
|
|
"loss": 6.0904669226147234e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.3003501296043396,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 907,
|
|
"train_speed(iter/s)": 0.01248
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 827.0,
|
|
"completions/mean_length": 279.3515625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.5265294288199478,
|
|
"grad_norm": 0.4505179327786,
|
|
"kl": 0.066162109375,
|
|
"learning_rate": 4.388094226967489e-07,
|
|
"loss": 6.620572821702808e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.68359375,
|
|
"reward_std": 0.1760166734457016,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8046875,
|
|
"rewards/VQAORM/std": 0.3979988098144531,
|
|
"step": 908,
|
|
"train_speed(iter/s)": 0.012488
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1123.0,
|
|
"completions/mean_length": 266.453125,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.5271093070455204,
|
|
"grad_norm": 0.3493379197506423,
|
|
"kl": 0.056884765625,
|
|
"learning_rate": 4.386644417919647e-07,
|
|
"loss": 5.698258974007331e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30078125,
|
|
"reward_std": 0.12189007550477982,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 909,
|
|
"train_speed(iter/s)": 0.012495
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 659.0,
|
|
"completions/mean_length": 265.90625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.5276891852710931,
|
|
"grad_norm": 0.6248959304900201,
|
|
"kl": 0.0577392578125,
|
|
"learning_rate": 4.385193164358622e-07,
|
|
"loss": 5.76767124584876e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37109375,
|
|
"reward_std": 0.2536258101463318,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 910,
|
|
"train_speed(iter/s)": 0.012504
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 780.0,
|
|
"completions/mean_length": 276.7578125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.5282690634966657,
|
|
"grad_norm": 0.583841516627684,
|
|
"kl": 0.063720703125,
|
|
"learning_rate": 4.3837404675653006e-07,
|
|
"loss": 6.37248158454895e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.36030110716819763,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 911,
|
|
"train_speed(iter/s)": 0.012512
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 831.0,
|
|
"completions/mean_length": 303.4375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.5288489417222383,
|
|
"grad_norm": 0.5249885913648915,
|
|
"kl": 0.0577392578125,
|
|
"learning_rate": 4.382286328821843e-07,
|
|
"loss": 5.7804947573458776e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.27019578218460083,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 912,
|
|
"train_speed(iter/s)": 0.01252
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 651.0,
|
|
"completions/mean_length": 271.4921875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.529428819947811,
|
|
"grad_norm": 0.46817278125028283,
|
|
"kl": 0.0609130859375,
|
|
"learning_rate": 4.380830749411681e-07,
|
|
"loss": 6.097492587286979e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.20618988573551178,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 913,
|
|
"train_speed(iter/s)": 0.012527
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 682.0,
|
|
"completions/mean_length": 289.484375,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.5300086981733836,
|
|
"grad_norm": 0.5698477190830255,
|
|
"kl": 0.063720703125,
|
|
"learning_rate": 4.379373730619519e-07,
|
|
"loss": 6.36627446510829e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.1869390904903412,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 914,
|
|
"train_speed(iter/s)": 0.012534
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 543.0,
|
|
"completions/mean_length": 262.8046875,
|
|
"completions/min_length": 71.0,
|
|
"epoch": 0.5305885763989562,
|
|
"grad_norm": 0.6685897612488547,
|
|
"kl": 0.06494140625,
|
|
"learning_rate": 4.3779152737313314e-07,
|
|
"loss": 6.504543125629425e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.703125,
|
|
"reward_std": 0.31707078218460083,
|
|
"rewards/CSTORM/mean": 0.390625,
|
|
"rewards/CSTORM/std": 0.20751149952411652,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 915,
|
|
"train_speed(iter/s)": 0.012542
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1565.0,
|
|
"completions/mean_length": 295.3515625,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.5311684546245289,
|
|
"grad_norm": 0.5857866681839531,
|
|
"kl": 0.0548095703125,
|
|
"learning_rate": 4.3764553800343617e-07,
|
|
"loss": 5.483954009832814e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.31640625,
|
|
"reward_std": 0.3016643524169922,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 916,
|
|
"train_speed(iter/s)": 0.012547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 841.0,
|
|
"completions/mean_length": 293.5859375,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.5317483328501015,
|
|
"grad_norm": 0.6139702366308201,
|
|
"kl": 0.058349609375,
|
|
"learning_rate": 4.374994050817121e-07,
|
|
"loss": 5.835421325173229e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.2890625,
|
|
"reward_std": 0.2728765904903412,
|
|
"rewards/CSTORM/mean": 0.234375,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 917,
|
|
"train_speed(iter/s)": 0.012555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 638.0,
|
|
"completions/mean_length": 265.71875,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.5323282110756741,
|
|
"grad_norm": 0.6433587350605571,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 4.37353128736939e-07,
|
|
"loss": 6.962910993024707e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.2777109742164612,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 918,
|
|
"train_speed(iter/s)": 0.012564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 879.0,
|
|
"completions/mean_length": 269.609375,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.5329080893012468,
|
|
"grad_norm": 0.5117167395204055,
|
|
"kl": 0.057861328125,
|
|
"learning_rate": 4.3720670909822115e-07,
|
|
"loss": 5.7873359764926136e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.24434107542037964,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 919,
|
|
"train_speed(iter/s)": 0.01257
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1000.0,
|
|
"completions/mean_length": 280.4296875,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.5334879675268194,
|
|
"grad_norm": 0.5078432019084568,
|
|
"kl": 0.058837890625,
|
|
"learning_rate": 4.370601462947896e-07,
|
|
"loss": 5.883361882297322e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.2910842299461365,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 920,
|
|
"train_speed(iter/s)": 0.012577
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 974.0,
|
|
"completions/mean_length": 258.546875,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.534067845752392,
|
|
"grad_norm": 0.6010451946323165,
|
|
"kl": 0.059814453125,
|
|
"learning_rate": 4.3691344045600167e-07,
|
|
"loss": 5.9794743719976395e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.2687046527862549,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 921,
|
|
"train_speed(iter/s)": 0.012585
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 664.0,
|
|
"completions/mean_length": 241.203125,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.5346477239779647,
|
|
"grad_norm": 0.7288691855190907,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 4.367665917113408e-07,
|
|
"loss": 6.593799480469897e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.27382156252861023,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 922,
|
|
"train_speed(iter/s)": 0.012594
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 958.0,
|
|
"completions/mean_length": 275.3125,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.5352276022035373,
|
|
"grad_norm": 0.6225763744893604,
|
|
"kl": 0.0650634765625,
|
|
"learning_rate": 4.366196001904167e-07,
|
|
"loss": 6.518274312838912e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.296875,
|
|
"reward_std": 0.23005647957324982,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 923,
|
|
"train_speed(iter/s)": 0.0126
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1080.0,
|
|
"completions/mean_length": 261.8984375,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.5358074804291099,
|
|
"grad_norm": 0.47663738698443847,
|
|
"kl": 0.0562744140625,
|
|
"learning_rate": 4.3647246602296506e-07,
|
|
"loss": 5.6264099839609116e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.294412761926651,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 924,
|
|
"train_speed(iter/s)": 0.012607
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 632.0,
|
|
"completions/mean_length": 272.96875,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.5363873586546826,
|
|
"grad_norm": 0.487851219277959,
|
|
"kl": 0.0618896484375,
|
|
"learning_rate": 4.363251893388476e-07,
|
|
"loss": 6.178152398206294e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.14170178771018982,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 925,
|
|
"train_speed(iter/s)": 0.012616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1014.0,
|
|
"completions/mean_length": 253.7890625,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.5369672368802552,
|
|
"grad_norm": 0.5160138122253154,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 4.3617777026805145e-07,
|
|
"loss": 6.792972999392077e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.27734375,
|
|
"reward_std": 0.16634789109230042,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 926,
|
|
"train_speed(iter/s)": 0.012623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1077.0,
|
|
"completions/mean_length": 266.0,
|
|
"completions/min_length": 60.0,
|
|
"epoch": 0.5375471151058278,
|
|
"grad_norm": 0.5437396968190313,
|
|
"kl": 0.0609130859375,
|
|
"learning_rate": 4.360302089406899e-07,
|
|
"loss": 6.094217678764835e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.23959344625473022,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 927,
|
|
"train_speed(iter/s)": 0.01263
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 669.0,
|
|
"completions/mean_length": 260.4453125,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.5381269933314005,
|
|
"grad_norm": 0.5523584390984971,
|
|
"kl": 0.0648193359375,
|
|
"learning_rate": 4.358825054870013e-07,
|
|
"loss": 6.47941924398765e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.280538409948349,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 928,
|
|
"train_speed(iter/s)": 0.012638
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 894.0,
|
|
"completions/mean_length": 277.0546875,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.538706871556973,
|
|
"grad_norm": 0.571882534086859,
|
|
"kl": 0.11669921875,
|
|
"learning_rate": 4.3573466003734993e-07,
|
|
"loss": 0.0001164556888397783,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.3532869815826416,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 929,
|
|
"train_speed(iter/s)": 0.012618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1564.0,
|
|
"completions/mean_length": 310.578125,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.5392867497825456,
|
|
"grad_norm": 0.6310678281528129,
|
|
"kl": 0.10546875,
|
|
"learning_rate": 4.3558667272222513e-07,
|
|
"loss": 0.00010533274326007813,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.2805572748184204,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 930,
|
|
"train_speed(iter/s)": 0.01261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/mean_length": 258.1328125,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.5398666280081182,
|
|
"grad_norm": 0.4778189403412792,
|
|
"kl": 0.1434326171875,
|
|
"learning_rate": 4.3543854367224124e-07,
|
|
"loss": 0.0001438008330296725,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.19721922278404236,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 931,
|
|
"train_speed(iter/s)": 0.01256
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1181.0,
|
|
"completions/mean_length": 283.484375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.540446506233691,
|
|
"grad_norm": 0.5253627762945502,
|
|
"kl": 0.05859375,
|
|
"learning_rate": 4.352902730181382e-07,
|
|
"loss": 5.856621282873675e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.31862977147102356,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 932,
|
|
"train_speed(iter/s)": 0.012566
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 781.0,
|
|
"completions/mean_length": 283.328125,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.5410263844592635,
|
|
"grad_norm": 0.4846080701088298,
|
|
"kl": 0.056396484375,
|
|
"learning_rate": 4.351418608907806e-07,
|
|
"loss": 5.636868445435539e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.26238328218460083,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 933,
|
|
"train_speed(iter/s)": 0.012575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 823.0,
|
|
"completions/mean_length": 282.515625,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.5416062626848361,
|
|
"grad_norm": 0.49087256546595753,
|
|
"kl": 0.05712890625,
|
|
"learning_rate": 4.3499330742115795e-07,
|
|
"loss": 5.719650653190911e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.2192658632993698,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 934,
|
|
"train_speed(iter/s)": 0.012583
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 755.0,
|
|
"completions/mean_length": 295.421875,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.5421861409104088,
|
|
"grad_norm": 0.56146476211287,
|
|
"kl": 0.051513671875,
|
|
"learning_rate": 4.348446127403845e-07,
|
|
"loss": 5.1484246796462685e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.2126619666814804,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 935,
|
|
"train_speed(iter/s)": 0.012591
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 657.0,
|
|
"completions/mean_length": 278.1875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.5427660191359814,
|
|
"grad_norm": 0.700028803920762,
|
|
"kl": 0.058349609375,
|
|
"learning_rate": 4.346957769796992e-07,
|
|
"loss": 5.834394687553868e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.41067010164260864,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 936,
|
|
"train_speed(iter/s)": 0.012599
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 729.0,
|
|
"completions/mean_length": 255.15625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.543345897361554,
|
|
"grad_norm": 0.6772840693500701,
|
|
"kl": 0.066650390625,
|
|
"learning_rate": 4.345468002704654e-07,
|
|
"loss": 6.656865298282355e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.3532869815826416,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 937,
|
|
"train_speed(iter/s)": 0.012608
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 948.0,
|
|
"completions/mean_length": 254.6640625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.5439257755871267,
|
|
"grad_norm": 0.6394938908900256,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 4.343976827441711e-07,
|
|
"loss": 6.80591692798771e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.34048938751220703,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 938,
|
|
"train_speed(iter/s)": 0.012615
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1715.0,
|
|
"completions/mean_length": 312.90625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.5445056538126993,
|
|
"grad_norm": 0.505171985628119,
|
|
"kl": 0.0556640625,
|
|
"learning_rate": 4.3424842453242825e-07,
|
|
"loss": 5.566840263782069e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.27557218074798584,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 939,
|
|
"train_speed(iter/s)": 0.012619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 781.0,
|
|
"completions/mean_length": 258.890625,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.5450855320382719,
|
|
"grad_norm": 0.5031135051084229,
|
|
"kl": 0.0626220703125,
|
|
"learning_rate": 4.3409902576697315e-07,
|
|
"loss": 6.26478431513533e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.20981568098068237,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 940,
|
|
"train_speed(iter/s)": 0.012625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 695.0,
|
|
"completions/mean_length": 251.2421875,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.5456654102638446,
|
|
"grad_norm": 0.4118923487399342,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 4.3394948657966624e-07,
|
|
"loss": 6.803940050303936e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.13630647957324982,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 941,
|
|
"train_speed(iter/s)": 0.012633
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1170.0,
|
|
"completions/mean_length": 283.2578125,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.5462452884894172,
|
|
"grad_norm": 0.5573680162484712,
|
|
"kl": 0.059326171875,
|
|
"learning_rate": 4.337998071024917e-07,
|
|
"loss": 5.9350397350499406e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.2926883101463318,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 942,
|
|
"train_speed(iter/s)": 0.012639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 739.0,
|
|
"completions/mean_length": 242.75,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.5468251667149898,
|
|
"grad_norm": 0.3291583010242686,
|
|
"kl": 0.068603515625,
|
|
"learning_rate": 4.336499874675576e-07,
|
|
"loss": 6.879944703541696e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.13898321986198425,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 943,
|
|
"train_speed(iter/s)": 0.012646
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 909.0,
|
|
"completions/mean_length": 281.796875,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.5474050449405625,
|
|
"grad_norm": 0.6169826102416585,
|
|
"kl": 0.0604248046875,
|
|
"learning_rate": 4.335000278070958e-07,
|
|
"loss": 6.0444097471190616e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.30654376745224,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 944,
|
|
"train_speed(iter/s)": 0.012652
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 591.0,
|
|
"completions/mean_length": 248.4140625,
|
|
"completions/min_length": 56.0,
|
|
"epoch": 0.5479849231661351,
|
|
"grad_norm": 0.5947375375449062,
|
|
"kl": 0.0654296875,
|
|
"learning_rate": 4.333499282534618e-07,
|
|
"loss": 6.554656283697113e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.28206315636634827,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 945,
|
|
"train_speed(iter/s)": 0.01266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 861.0,
|
|
"completions/mean_length": 280.1015625,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.5485648013917077,
|
|
"grad_norm": 0.6328818005072991,
|
|
"kl": 0.0567626953125,
|
|
"learning_rate": 4.3319968893913425e-07,
|
|
"loss": 5.6701752328081056e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.2066001147031784,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 946,
|
|
"train_speed(iter/s)": 0.012656
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 881.0,
|
|
"completions/mean_length": 266.421875,
|
|
"completions/min_length": 68.0,
|
|
"epoch": 0.5491446796172804,
|
|
"grad_norm": 0.4887060874437861,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 4.330493099967154e-07,
|
|
"loss": 6.985447544138879e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.24106568098068237,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 947,
|
|
"train_speed(iter/s)": 0.012664
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 759.0,
|
|
"completions/mean_length": 257.7265625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.549724557842853,
|
|
"grad_norm": 0.6324059284420304,
|
|
"kl": 0.058837890625,
|
|
"learning_rate": 4.328987915589308e-07,
|
|
"loss": 5.8748079027282074e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.27734375,
|
|
"reward_std": 0.32821178436279297,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.53125,
|
|
"rewards/VQAORM/std": 0.5009832978248596,
|
|
"step": 948,
|
|
"train_speed(iter/s)": 0.012672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 558.0,
|
|
"completions/mean_length": 253.625,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.5503044360684256,
|
|
"grad_norm": 0.5298213385324664,
|
|
"kl": 0.0576171875,
|
|
"learning_rate": 4.3274813375862903e-07,
|
|
"loss": 5.763339140685275e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.26721763610839844,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 949,
|
|
"train_speed(iter/s)": 0.012681
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 622.0,
|
|
"completions/mean_length": 269.6796875,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.5508843142939983,
|
|
"grad_norm": 0.6522329444872879,
|
|
"kl": 0.06103515625,
|
|
"learning_rate": 4.325973367287816e-07,
|
|
"loss": 6.109044625191018e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.294412761926651,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 950,
|
|
"train_speed(iter/s)": 0.012688
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 626.0,
|
|
"completions/mean_length": 272.53125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.5514641925195709,
|
|
"grad_norm": 0.4809851377813423,
|
|
"kl": 0.056396484375,
|
|
"learning_rate": 4.324464006024829e-07,
|
|
"loss": 5.636006244458258e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.2529141902923584,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 951,
|
|
"train_speed(iter/s)": 0.012696
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 865.0,
|
|
"completions/mean_length": 285.3828125,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.5520440707451435,
|
|
"grad_norm": 0.47701737109719317,
|
|
"kl": 0.0614013671875,
|
|
"learning_rate": 4.322953255129503e-07,
|
|
"loss": 6.146984378574416e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.2013554871082306,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 952,
|
|
"train_speed(iter/s)": 0.012702
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 745.0,
|
|
"completions/mean_length": 250.03125,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.5526239489707161,
|
|
"grad_norm": 0.6270777499437973,
|
|
"kl": 0.0703125,
|
|
"learning_rate": 4.321441115935235e-07,
|
|
"loss": 7.024469960015267e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.34469497203826904,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 953,
|
|
"train_speed(iter/s)": 0.01271
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 596.0,
|
|
"completions/mean_length": 250.109375,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.5532038271962888,
|
|
"grad_norm": 0.5638540127903799,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.31992758977665e-07,
|
|
"loss": 6.936003046575934e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.2871611714363098,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 954,
|
|
"train_speed(iter/s)": 0.012717
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 664.0,
|
|
"completions/mean_length": 273.25,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.5537837054218614,
|
|
"grad_norm": 0.6609620202101308,
|
|
"kl": 0.064208984375,
|
|
"learning_rate": 4.318412677989596e-07,
|
|
"loss": 6.433062662836164e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3359375,
|
|
"reward_std": 0.2885015904903412,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 955,
|
|
"train_speed(iter/s)": 0.012725
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 714.0,
|
|
"completions/mean_length": 241.2890625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.554363583647434,
|
|
"grad_norm": 0.628439833913807,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.316896381911145e-07,
|
|
"loss": 6.939360173419118e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.23247367143630981,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 956,
|
|
"train_speed(iter/s)": 0.012734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 536.0,
|
|
"completions/mean_length": 246.65625,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.5549434618730067,
|
|
"grad_norm": 0.5844884601087684,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 4.315378702879589e-07,
|
|
"loss": 6.965330976527184e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.2755722105503082,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 957,
|
|
"train_speed(iter/s)": 0.012742
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 694.0,
|
|
"completions/mean_length": 247.453125,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.5555233400985793,
|
|
"grad_norm": 0.48980406097273355,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.313859642234442e-07,
|
|
"loss": 6.938685692148283e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.609375,
|
|
"reward_std": 0.1814119815826416,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 958,
|
|
"train_speed(iter/s)": 0.01275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 647.0,
|
|
"completions/mean_length": 245.3828125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.5561032183241519,
|
|
"grad_norm": 0.6185610635998078,
|
|
"kl": 0.0660400390625,
|
|
"learning_rate": 4.312339201316436e-07,
|
|
"loss": 6.602160283364356e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.2806890904903412,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 959,
|
|
"train_speed(iter/s)": 0.012758
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 817.0,
|
|
"completions/mean_length": 257.703125,
|
|
"completions/min_length": 62.0,
|
|
"epoch": 0.5566830965497246,
|
|
"grad_norm": 0.4379623549061326,
|
|
"kl": 0.06396484375,
|
|
"learning_rate": 4.3108173814675235e-07,
|
|
"loss": 6.404919986380264e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.17670938372612,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 960,
|
|
"train_speed(iter/s)": 0.012766
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 839.0,
|
|
"completions/mean_length": 298.671875,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.5572629747752972,
|
|
"grad_norm": 0.5406750644585466,
|
|
"kl": 0.0531005859375,
|
|
"learning_rate": 4.309294184030872e-07,
|
|
"loss": 5.311657878337428e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.27289140224456787,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 961,
|
|
"train_speed(iter/s)": 0.012772
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 757.0,
|
|
"completions/mean_length": 261.65625,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.5578428530008698,
|
|
"grad_norm": 0.5951430669174033,
|
|
"kl": 0.0635986328125,
|
|
"learning_rate": 4.3077696103508663e-07,
|
|
"loss": 6.363834836520255e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.30925828218460083,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 962,
|
|
"train_speed(iter/s)": 0.01278
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 552.0,
|
|
"completions/mean_length": 273.5,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.5584227312264425,
|
|
"grad_norm": 0.6581977036420399,
|
|
"kl": 0.055908203125,
|
|
"learning_rate": 4.306243661773105e-07,
|
|
"loss": 5.5923355830600485e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.3555534780025482,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 963,
|
|
"train_speed(iter/s)": 0.012788
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 724.0,
|
|
"completions/mean_length": 264.6484375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.5590026094520151,
|
|
"grad_norm": 0.6627189850288441,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 4.3047163396444e-07,
|
|
"loss": 6.90316956024617e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.39563295245170593,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 964,
|
|
"train_speed(iter/s)": 0.012795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 873.0,
|
|
"completions/mean_length": 250.4921875,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.5595824876775877,
|
|
"grad_norm": 0.594214687996011,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 4.303187645312775e-07,
|
|
"loss": 6.608545663766563e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.2223757952451706,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 965,
|
|
"train_speed(iter/s)": 0.012803
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 898.0,
|
|
"completions/mean_length": 291.4921875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.5601623659031604,
|
|
"grad_norm": 0.5564461671809823,
|
|
"kl": 0.0596923828125,
|
|
"learning_rate": 4.301657580127468e-07,
|
|
"loss": 5.968528057564981e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.2456626147031784,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 966,
|
|
"train_speed(iter/s)": 0.012811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 634.0,
|
|
"completions/mean_length": 234.2734375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.560742244128733,
|
|
"grad_norm": 0.5831209984354657,
|
|
"kl": 0.086181640625,
|
|
"learning_rate": 4.3001261454389227e-07,
|
|
"loss": 8.59992578625679e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.640625,
|
|
"reward_std": 0.270975261926651,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 967,
|
|
"train_speed(iter/s)": 0.01282
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 894.0,
|
|
"completions/mean_length": 281.6484375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.5613221223543056,
|
|
"grad_norm": 0.751903602454995,
|
|
"kl": 0.0594482421875,
|
|
"learning_rate": 4.298593342598795e-07,
|
|
"loss": 5.945655721006915e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.4116002917289734,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 968,
|
|
"train_speed(iter/s)": 0.012826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 705.0,
|
|
"completions/mean_length": 238.140625,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.5619020005798783,
|
|
"grad_norm": 0.44535656524353495,
|
|
"kl": 0.073486328125,
|
|
"learning_rate": 4.297059172959945e-07,
|
|
"loss": 7.358190487138927e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.15314006805419922,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 969,
|
|
"train_speed(iter/s)": 0.012834
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1839.0,
|
|
"completions/mean_length": 300.0859375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.5624818788054509,
|
|
"grad_norm": 0.49762899717278103,
|
|
"kl": 0.0623779296875,
|
|
"learning_rate": 4.2955236378764437e-07,
|
|
"loss": 6.238582136575133e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62890625,
|
|
"reward_std": 0.1873493194580078,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 970,
|
|
"train_speed(iter/s)": 0.012824
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 758.0,
|
|
"completions/mean_length": 260.8046875,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.5630617570310235,
|
|
"grad_norm": 0.5868164601474167,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 4.293986738703562e-07,
|
|
"loss": 6.6975990193896e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.30598288774490356,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 971,
|
|
"train_speed(iter/s)": 0.012832
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 814.0,
|
|
"completions/mean_length": 260.8125,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.5636416352565962,
|
|
"grad_norm": 0.6673471533922065,
|
|
"kl": 0.06494140625,
|
|
"learning_rate": 4.2924484767977777e-07,
|
|
"loss": 6.492157990578562e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62109375,
|
|
"reward_std": 0.2746010422706604,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 972,
|
|
"train_speed(iter/s)": 0.01284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 556.0,
|
|
"completions/mean_length": 230.25,
|
|
"completions/min_length": 66.0,
|
|
"epoch": 0.5642215134821688,
|
|
"grad_norm": 0.5989430375111489,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 4.290908853516771e-07,
|
|
"loss": 6.700629455735907e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.25213468074798584,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 973,
|
|
"train_speed(iter/s)": 0.012849
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 841.0,
|
|
"completions/mean_length": 276.734375,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.5648013917077414,
|
|
"grad_norm": 0.50934083011177,
|
|
"kl": 0.0616455078125,
|
|
"learning_rate": 4.289367870219424e-07,
|
|
"loss": 6.154316361062229e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.1725226789712906,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 974,
|
|
"train_speed(iter/s)": 0.012857
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 770.0,
|
|
"completions/mean_length": 291.3671875,
|
|
"completions/min_length": 119.0,
|
|
"epoch": 0.5653812699333141,
|
|
"grad_norm": 0.5598725717116144,
|
|
"kl": 0.05712890625,
|
|
"learning_rate": 4.287825528265819e-07,
|
|
"loss": 5.712195343221538e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37109375,
|
|
"reward_std": 0.3158244490623474,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 975,
|
|
"train_speed(iter/s)": 0.012865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 692.0,
|
|
"completions/mean_length": 287.765625,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.5659611481588867,
|
|
"grad_norm": 0.5087752844070148,
|
|
"kl": 0.0615234375,
|
|
"learning_rate": 4.286281829017237e-07,
|
|
"loss": 6.162106728879735e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.2493072748184204,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 976,
|
|
"train_speed(iter/s)": 0.012872
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 606.0,
|
|
"completions/mean_length": 233.984375,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.5665410263844592,
|
|
"grad_norm": 0.39760581282472657,
|
|
"kl": 0.06494140625,
|
|
"learning_rate": 4.2847367738361574e-07,
|
|
"loss": 6.513650441775098e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.75390625,
|
|
"reward_std": 0.16634789109230042,
|
|
"rewards/CSTORM/mean": 0.41015625,
|
|
"rewards/CSTORM/std": 0.1927177608013153,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.84375,
|
|
"rewards/VQAORM/std": 0.3645188808441162,
|
|
"step": 977,
|
|
"train_speed(iter/s)": 0.01288
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 689.0,
|
|
"completions/mean_length": 262.9140625,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.5671209046100318,
|
|
"grad_norm": 0.5191536089988934,
|
|
"kl": 0.06005859375,
|
|
"learning_rate": 4.2831903640862554e-07,
|
|
"loss": 6.0066238802392036e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.21484375,
|
|
"reward_std": 0.2848758101463318,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 978,
|
|
"train_speed(iter/s)": 0.012888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 754.0,
|
|
"completions/mean_length": 262.5078125,
|
|
"completions/min_length": 128.0,
|
|
"epoch": 0.5677007828356045,
|
|
"grad_norm": 0.6067118736213475,
|
|
"kl": 0.0648193359375,
|
|
"learning_rate": 4.2816426011324024e-07,
|
|
"loss": 6.482356548076496e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.2378501147031784,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 979,
|
|
"train_speed(iter/s)": 0.012896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 601.0,
|
|
"completions/mean_length": 235.7578125,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.5682806610611771,
|
|
"grad_norm": 0.6115661487323062,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.2800934863406647e-07,
|
|
"loss": 6.933163240319118e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.19596019387245178,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 980,
|
|
"train_speed(iter/s)": 0.012904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 730.0,
|
|
"completions/mean_length": 251.0859375,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.5688605392867497,
|
|
"grad_norm": 0.5311178897069407,
|
|
"kl": 0.0654296875,
|
|
"learning_rate": 4.2785430210783015e-07,
|
|
"loss": 6.552496779477224e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.237308070063591,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 981,
|
|
"train_speed(iter/s)": 0.012911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 575.0,
|
|
"completions/mean_length": 253.875,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.5694404175123224,
|
|
"grad_norm": 0.5411533545168843,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 4.2769912067137624e-07,
|
|
"loss": 6.578353350050747e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.2536216974258423,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 982,
|
|
"train_speed(iter/s)": 0.012919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 968.0,
|
|
"completions/mean_length": 260.2578125,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.570020295737895,
|
|
"grad_norm": 0.5592055558847409,
|
|
"kl": 0.0634765625,
|
|
"learning_rate": 4.275438044616691e-07,
|
|
"loss": 6.355716323014349e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.23139688372612,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 983,
|
|
"train_speed(iter/s)": 0.012926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 881.0,
|
|
"completions/mean_length": 299.2421875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.5706001739634676,
|
|
"grad_norm": 0.6205711000361664,
|
|
"kl": 0.0533447265625,
|
|
"learning_rate": 4.273883536157917e-07,
|
|
"loss": 5.3292598749976605e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.296875,
|
|
"reward_std": 0.4001619815826416,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 984,
|
|
"train_speed(iter/s)": 0.012934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 666.0,
|
|
"completions/mean_length": 285.1640625,
|
|
"completions/min_length": 58.0,
|
|
"epoch": 0.5711800521890403,
|
|
"grad_norm": 0.6427456028658284,
|
|
"kl": 0.0633544921875,
|
|
"learning_rate": 4.2723276827094607e-07,
|
|
"loss": 6.336097430903465e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.3280611038208008,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 985,
|
|
"train_speed(iter/s)": 0.012942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1245.0,
|
|
"completions/mean_length": 257.0625,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.5717599304146129,
|
|
"grad_norm": 0.4068517251284017,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 4.2707704856445277e-07,
|
|
"loss": 6.680291699012741e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.75,
|
|
"reward_std": 0.1478765904903412,
|
|
"rewards/CSTORM/mean": 0.4140625,
|
|
"rewards/CSTORM/std": 0.1893770843744278,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8359375,
|
|
"rewards/VQAORM/std": 0.371787428855896,
|
|
"step": 986,
|
|
"train_speed(iter/s)": 0.012948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1150.0,
|
|
"completions/mean_length": 273.8828125,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.5723398086401855,
|
|
"grad_norm": 0.5413291264529659,
|
|
"kl": 0.060302734375,
|
|
"learning_rate": 4.269211946337511e-07,
|
|
"loss": 6.0355036112014204e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.28580188751220703,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 987,
|
|
"train_speed(iter/s)": 0.012952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 760.0,
|
|
"completions/mean_length": 276.078125,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.5729196868657582,
|
|
"grad_norm": 0.46326484100977455,
|
|
"kl": 0.0614013671875,
|
|
"learning_rate": 4.267652066163988e-07,
|
|
"loss": 6.133341230452061e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3515625,
|
|
"reward_std": 0.171875,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 988,
|
|
"train_speed(iter/s)": 0.01296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 807.0,
|
|
"completions/mean_length": 273.546875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.5734995650913308,
|
|
"grad_norm": 0.5144770281666089,
|
|
"kl": 0.0609130859375,
|
|
"learning_rate": 4.2660908465007184e-07,
|
|
"loss": 6.091555405873805e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.22090357542037964,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 989,
|
|
"train_speed(iter/s)": 0.012967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 690.0,
|
|
"completions/mean_length": 269.5078125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.5740794433169034,
|
|
"grad_norm": 0.49237249648557335,
|
|
"kl": 0.06201171875,
|
|
"learning_rate": 4.2645282887256464e-07,
|
|
"loss": 6.197337643243372e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.1635015904903412,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 990,
|
|
"train_speed(iter/s)": 0.012976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 801.0,
|
|
"completions/mean_length": 261.0078125,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.5746593215424761,
|
|
"grad_norm": 0.54796783599905,
|
|
"kl": 0.061279296875,
|
|
"learning_rate": 4.2629643942178937e-07,
|
|
"loss": 6.118077726569027e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.57421875,
|
|
"reward_std": 0.21322289109230042,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 991,
|
|
"train_speed(iter/s)": 0.012984
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 563.0,
|
|
"completions/mean_length": 224.03125,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.5752391997680487,
|
|
"grad_norm": 0.5904052703904349,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 4.261399164357766e-07,
|
|
"loss": 7.049052510410547e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.28204429149627686,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 992,
|
|
"train_speed(iter/s)": 0.012992
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 861.0,
|
|
"completions/mean_length": 247.75,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.5758190779936213,
|
|
"grad_norm": 0.5810404341668541,
|
|
"kl": 0.073486328125,
|
|
"learning_rate": 4.2598326005267436e-07,
|
|
"loss": 7.335165719268844e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.2529330551624298,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 993,
|
|
"train_speed(iter/s)": 0.012999
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 577.0,
|
|
"completions/mean_length": 258.6015625,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.576398956219194,
|
|
"grad_norm": 0.5141281226256932,
|
|
"kl": 0.09814453125,
|
|
"learning_rate": 4.258264704107488e-07,
|
|
"loss": 9.841016435530037e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.2360994815826416,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 994,
|
|
"train_speed(iter/s)": 0.013008
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 534.0,
|
|
"completions/mean_length": 245.5234375,
|
|
"completions/min_length": 120.0,
|
|
"epoch": 0.5769788344447666,
|
|
"grad_norm": 0.5202577781063713,
|
|
"kl": 0.069091796875,
|
|
"learning_rate": 4.2566954764838336e-07,
|
|
"loss": 6.920320447534323e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.19880647957324982,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 995,
|
|
"train_speed(iter/s)": 0.013016
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 901.0,
|
|
"completions/mean_length": 267.3515625,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.5775587126703392,
|
|
"grad_norm": 0.5860657012025676,
|
|
"kl": 0.0662841796875,
|
|
"learning_rate": 4.2551249190407917e-07,
|
|
"loss": 6.627410766668618e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59765625,
|
|
"reward_std": 0.2677597105503082,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 996,
|
|
"train_speed(iter/s)": 0.013022
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 733.0,
|
|
"completions/mean_length": 275.546875,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.5781385908959119,
|
|
"grad_norm": 0.5476105465944336,
|
|
"kl": 0.064208984375,
|
|
"learning_rate": 4.2535530331645463e-07,
|
|
"loss": 6.415945244953036e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.23005646467208862,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 997,
|
|
"train_speed(iter/s)": 0.013031
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 689.0,
|
|
"completions/mean_length": 240.3515625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.5787184691214845,
|
|
"grad_norm": 0.6252967504397297,
|
|
"kl": 0.072509765625,
|
|
"learning_rate": 4.251979820242454e-07,
|
|
"loss": 7.249507325468585e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.2777109742164612,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 998,
|
|
"train_speed(iter/s)": 0.013037
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 624.0,
|
|
"completions/mean_length": 253.046875,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.5792983473470571,
|
|
"grad_norm": 0.5411128924271346,
|
|
"kl": 0.0633544921875,
|
|
"learning_rate": 4.250405281663045e-07,
|
|
"loss": 6.350952025968581e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.24809867143630981,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 999,
|
|
"train_speed(iter/s)": 0.013045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 593.0,
|
|
"completions/mean_length": 239.8515625,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.5798782255726298,
|
|
"grad_norm": 0.6592748502849055,
|
|
"kl": 0.077392578125,
|
|
"learning_rate": 4.2488294188160153e-07,
|
|
"loss": 7.735897816019133e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62109375,
|
|
"reward_std": 0.2789195775985718,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1000,
|
|
"train_speed(iter/s)": 0.013054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1025.0,
|
|
"completions/mean_length": 261.4140625,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.5804581037982024,
|
|
"grad_norm": 0.4524796086484601,
|
|
"kl": 0.063232421875,
|
|
"learning_rate": 4.2472522330922314e-07,
|
|
"loss": 6.33462259429507e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.18978539109230042,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1001,
|
|
"train_speed(iter/s)": 0.013056
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 776.0,
|
|
"completions/mean_length": 256.875,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.581037982023775,
|
|
"grad_norm": 0.4362315498895984,
|
|
"kl": 0.0703125,
|
|
"learning_rate": 4.24567372588373e-07,
|
|
"loss": 7.037961040623486e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.7421875,
|
|
"reward_std": 0.14127269387245178,
|
|
"rewards/CSTORM/mean": 0.40625,
|
|
"rewards/CSTORM/std": 0.19592301547527313,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8359375,
|
|
"rewards/VQAORM/std": 0.371787428855896,
|
|
"step": 1002,
|
|
"train_speed(iter/s)": 0.013064
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 679.0,
|
|
"completions/mean_length": 275.96875,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.5816178602493476,
|
|
"grad_norm": 0.5531624501822273,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 4.244093898583711e-07,
|
|
"loss": 6.46636908641085e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.25498101115226746,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1003,
|
|
"train_speed(iter/s)": 0.013071
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1602.0,
|
|
"completions/mean_length": 293.1640625,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.5821977384749203,
|
|
"grad_norm": 0.5648951836571582,
|
|
"kl": 0.096435546875,
|
|
"learning_rate": 4.24251275258654e-07,
|
|
"loss": 9.630571003071964e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.28567010164260864,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1004,
|
|
"train_speed(iter/s)": 0.01302
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 982.0,
|
|
"completions/mean_length": 286.0859375,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.5827776167004929,
|
|
"grad_norm": 0.3947001309093183,
|
|
"kl": 0.0540771484375,
|
|
"learning_rate": 4.240930289287747e-07,
|
|
"loss": 5.4168318456504494e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.1741415113210678,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1005,
|
|
"train_speed(iter/s)": 0.013026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 551.0,
|
|
"completions/mean_length": 239.375,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.5833574949260655,
|
|
"grad_norm": 0.5500366033188065,
|
|
"kl": 0.07470703125,
|
|
"learning_rate": 4.239346510084024e-07,
|
|
"loss": 7.472735160263255e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.21939769387245178,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1006,
|
|
"train_speed(iter/s)": 0.013034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1330.0,
|
|
"completions/mean_length": 275.625,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.5839373731516382,
|
|
"grad_norm": 0.44107851784377167,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 4.2377614163732266e-07,
|
|
"loss": 6.68606226099655e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.2103765904903412,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1007,
|
|
"train_speed(iter/s)": 0.01304
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1157.0,
|
|
"completions/mean_length": 256.03125,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.5845172513772108,
|
|
"grad_norm": 0.37414380321975116,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 4.236175009554369e-07,
|
|
"loss": 7.008502871030942e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37109375,
|
|
"reward_std": 0.12685628235340118,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1008,
|
|
"train_speed(iter/s)": 0.013046
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 747.0,
|
|
"completions/mean_length": 268.0546875,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.5850971296027834,
|
|
"grad_norm": 0.5213373217859364,
|
|
"kl": 0.065185546875,
|
|
"learning_rate": 4.2345872910276225e-07,
|
|
"loss": 6.526726065203547e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.2451017051935196,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1009,
|
|
"train_speed(iter/s)": 0.013054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 514.0,
|
|
"completions/mean_length": 253.7734375,
|
|
"completions/min_length": 125.0,
|
|
"epoch": 0.5856770078283561,
|
|
"grad_norm": 0.5057850431333146,
|
|
"kl": 0.06396484375,
|
|
"learning_rate": 4.2329982621943187e-07,
|
|
"loss": 6.397024844773114e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.2145632952451706,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1010,
|
|
"train_speed(iter/s)": 0.013062
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 693.0,
|
|
"completions/mean_length": 262.0625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.5862568860539287,
|
|
"grad_norm": 0.5958264941223099,
|
|
"kl": 0.065673828125,
|
|
"learning_rate": 4.2314079244569463e-07,
|
|
"loss": 6.571388803422451e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.2427033632993698,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1011,
|
|
"train_speed(iter/s)": 0.013071
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 673.0,
|
|
"completions/mean_length": 288.2109375,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.5868367642795013,
|
|
"grad_norm": 0.5684143708024468,
|
|
"kl": 0.064208984375,
|
|
"learning_rate": 4.2298162792191454e-07,
|
|
"loss": 6.414022936951369e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.3340361714363098,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1012,
|
|
"train_speed(iter/s)": 0.013078
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 802.0,
|
|
"completions/mean_length": 262.6171875,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.587416642505074,
|
|
"grad_norm": 0.6246116449143377,
|
|
"kl": 0.065673828125,
|
|
"learning_rate": 4.2282233278857154e-07,
|
|
"loss": 6.554758874699473e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.30477428436279297,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1013,
|
|
"train_speed(iter/s)": 0.013086
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 607.0,
|
|
"completions/mean_length": 235.2265625,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.5879965207306466,
|
|
"grad_norm": 0.520133656394273,
|
|
"kl": 0.073974609375,
|
|
"learning_rate": 4.2266290718626033e-07,
|
|
"loss": 7.400385948130861e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.1970369666814804,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1014,
|
|
"train_speed(iter/s)": 0.013095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 719.0,
|
|
"completions/mean_length": 260.0625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.5885763989562192,
|
|
"grad_norm": 0.6935697870114347,
|
|
"kl": 0.0662841796875,
|
|
"learning_rate": 4.225033512556912e-07,
|
|
"loss": 6.618745101150125e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.3870859742164612,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1015,
|
|
"train_speed(iter/s)": 0.013102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 581.0,
|
|
"completions/mean_length": 233.8359375,
|
|
"completions/min_length": 71.0,
|
|
"epoch": 0.5891562771817919,
|
|
"grad_norm": 0.4760955060344682,
|
|
"kl": 0.066650390625,
|
|
"learning_rate": 4.2234366513768914e-07,
|
|
"loss": 6.659195059910417e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.24106568098068237,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1016,
|
|
"train_speed(iter/s)": 0.01311
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 612.0,
|
|
"completions/mean_length": 255.703125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.5897361554073645,
|
|
"grad_norm": 0.5207060923519019,
|
|
"kl": 0.0732421875,
|
|
"learning_rate": 4.2218384897319413e-07,
|
|
"loss": 7.337862916756421e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.2506476938724518,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1017,
|
|
"train_speed(iter/s)": 0.013117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 587.0,
|
|
"completions/mean_length": 260.03125,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.590316033632937,
|
|
"grad_norm": 0.5922055444912069,
|
|
"kl": 0.0625,
|
|
"learning_rate": 4.220239029032611e-07,
|
|
"loss": 6.24699387117289e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3671875,
|
|
"reward_std": 0.28163403272628784,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1018,
|
|
"train_speed(iter/s)": 0.013125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 965.0,
|
|
"completions/mean_length": 277.4609375,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.5908959118585098,
|
|
"grad_norm": 0.5330149818889837,
|
|
"kl": 0.0596923828125,
|
|
"learning_rate": 4.2186382706905956e-07,
|
|
"loss": 5.980241621728055e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.27192753553390503,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1019,
|
|
"train_speed(iter/s)": 0.013132
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 272.7578125,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.5914757900840824,
|
|
"grad_norm": 0.6273878208473149,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 4.2170362161187323e-07,
|
|
"loss": 6.651518924627453e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6640625,
|
|
"reward_std": 0.2700807452201843,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.8046875,
|
|
"rewards/VQAORM/std": 0.3979988098144531,
|
|
"step": 1020,
|
|
"train_speed(iter/s)": 0.013071
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 753.0,
|
|
"completions/mean_length": 249.8515625,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.592055668309655,
|
|
"grad_norm": 0.6402608198207795,
|
|
"kl": 0.068359375,
|
|
"learning_rate": 4.215432866731007e-07,
|
|
"loss": 6.828103505540639e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.33062899112701416,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1021,
|
|
"train_speed(iter/s)": 0.013079
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1352.0,
|
|
"completions/mean_length": 276.8203125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.5926355465352277,
|
|
"grad_norm": 0.49941248431437846,
|
|
"kl": 0.060791015625,
|
|
"learning_rate": 4.213828223942547e-07,
|
|
"loss": 6.078819205868058e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.22548164427280426,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1022,
|
|
"train_speed(iter/s)": 0.013084
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 720.0,
|
|
"completions/mean_length": 263.8984375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.5932154247608002,
|
|
"grad_norm": 0.6627479310401467,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 4.2122222891696197e-07,
|
|
"loss": 6.596023013116792e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.36969149112701416,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1023,
|
|
"train_speed(iter/s)": 0.01309
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 900.0,
|
|
"completions/mean_length": 250.984375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.5937953029863728,
|
|
"grad_norm": 0.48059984671207695,
|
|
"kl": 0.067138671875,
|
|
"learning_rate": 4.2106150638296334e-07,
|
|
"loss": 6.713617040077224e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.70703125,
|
|
"reward_std": 0.20944641530513763,
|
|
"rewards/CSTORM/mean": 0.39453125,
|
|
"rewards/CSTORM/std": 0.20478858053684235,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 1024,
|
|
"train_speed(iter/s)": 0.013097
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 561.0,
|
|
"completions/mean_length": 226.0859375,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.5943751812119455,
|
|
"grad_norm": 0.5086229824129072,
|
|
"kl": 0.0638427734375,
|
|
"learning_rate": 4.209006549341136e-07,
|
|
"loss": 6.390315684257075e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.15490958094596863,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1025,
|
|
"train_speed(iter/s)": 0.013106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 755.0,
|
|
"completions/mean_length": 254.6484375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.5949550594375181,
|
|
"grad_norm": 0.6544870549448017,
|
|
"kl": 0.0623779296875,
|
|
"learning_rate": 4.2073967471238137e-07,
|
|
"loss": 6.240390939638019e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.3062613010406494,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 1026,
|
|
"train_speed(iter/s)": 0.013114
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1049.0,
|
|
"completions/mean_length": 327.3515625,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.5955349376630907,
|
|
"grad_norm": 0.6354111352755266,
|
|
"kl": 0.054443359375,
|
|
"learning_rate": 4.205785658598489e-07,
|
|
"loss": 5.446535215014592e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.33928078413009644,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1027,
|
|
"train_speed(iter/s)": 0.013105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 642.0,
|
|
"completions/mean_length": 254.53125,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.5961148158886633,
|
|
"grad_norm": 0.445376728402849,
|
|
"kl": 0.0633544921875,
|
|
"learning_rate": 4.2041732851871165e-07,
|
|
"loss": 6.348081660689786e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62890625,
|
|
"reward_std": 0.1442507952451706,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1028,
|
|
"train_speed(iter/s)": 0.013112
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1151.0,
|
|
"completions/mean_length": 286.0078125,
|
|
"completions/min_length": 130.0,
|
|
"epoch": 0.596694694114236,
|
|
"grad_norm": 0.5682821951243971,
|
|
"kl": 0.0623779296875,
|
|
"learning_rate": 4.2025596283127914e-07,
|
|
"loss": 6.240561924641952e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2728765904903412,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1029,
|
|
"train_speed(iter/s)": 0.013118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 723.0,
|
|
"completions/mean_length": 250.96875,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.5972745723398086,
|
|
"grad_norm": 0.630637709830026,
|
|
"kl": 0.062255859375,
|
|
"learning_rate": 4.2009446893997354e-07,
|
|
"loss": 6.219941860763356e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.28878000378608704,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 1030,
|
|
"train_speed(iter/s)": 0.013126
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 679.0,
|
|
"completions/mean_length": 253.4765625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.5978544505653812,
|
|
"grad_norm": 0.504002055733181,
|
|
"kl": 0.063232421875,
|
|
"learning_rate": 4.199328469873305e-07,
|
|
"loss": 6.342872075038031e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.22181488573551178,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1031,
|
|
"train_speed(iter/s)": 0.013134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 509.0,
|
|
"completions/mean_length": 223.453125,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.5984343287909539,
|
|
"grad_norm": 0.566679296863793,
|
|
"kl": 0.074462890625,
|
|
"learning_rate": 4.1977109711599864e-07,
|
|
"loss": 7.436770829372108e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.25966876745224,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1032,
|
|
"train_speed(iter/s)": 0.013141
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 697.0,
|
|
"completions/mean_length": 277.078125,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.5990142070165265,
|
|
"grad_norm": 0.47317232669457815,
|
|
"kl": 0.0592041015625,
|
|
"learning_rate": 4.196092194687394e-07,
|
|
"loss": 5.91888529015705e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.25315165519714355,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1033,
|
|
"train_speed(iter/s)": 0.013149
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 865.0,
|
|
"completions/mean_length": 258.421875,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.5995940852420991,
|
|
"grad_norm": 0.5406335883389706,
|
|
"kl": 0.0631103515625,
|
|
"learning_rate": 4.194472141884271e-07,
|
|
"loss": 6.308940646704286e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.2338140904903412,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1034,
|
|
"train_speed(iter/s)": 0.013156
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1009.0,
|
|
"completions/mean_length": 270.5078125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.6001739634676718,
|
|
"grad_norm": 0.5696702701161083,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 4.192850814180487e-07,
|
|
"loss": 6.457559356931597e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.54296875,
|
|
"reward_std": 0.2025640904903412,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1035,
|
|
"train_speed(iter/s)": 0.013162
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 817.0,
|
|
"completions/mean_length": 263.796875,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.6007538416932444,
|
|
"grad_norm": 0.53895126113696,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 4.1912282130070366e-07,
|
|
"loss": 6.689049769192934e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.28375399112701416,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1036,
|
|
"train_speed(iter/s)": 0.01317
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 795.0,
|
|
"completions/mean_length": 256.6015625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.601333719918817,
|
|
"grad_norm": 0.5649424257532546,
|
|
"kl": 0.06494140625,
|
|
"learning_rate": 4.189604339796038e-07,
|
|
"loss": 6.494616536656395e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.21684867143630981,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1037,
|
|
"train_speed(iter/s)": 0.013177
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 852.0,
|
|
"completions/mean_length": 239.96875,
|
|
"completions/min_length": 47.0,
|
|
"epoch": 0.6019135981443897,
|
|
"grad_norm": 0.49499806368213595,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 4.187979195980732e-07,
|
|
"loss": 7.061110954964533e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.21521097421646118,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1038,
|
|
"train_speed(iter/s)": 0.013185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 754.0,
|
|
"completions/mean_length": 271.28125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.6024934763699623,
|
|
"grad_norm": 0.49995726895853626,
|
|
"kl": 0.060546875,
|
|
"learning_rate": 4.186352782995482e-07,
|
|
"loss": 6.046720955055207e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.3275640904903412,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 1039,
|
|
"train_speed(iter/s)": 0.013192
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 858.0,
|
|
"completions/mean_length": 253.546875,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.6030733545955349,
|
|
"grad_norm": 0.4846090866593313,
|
|
"kl": 0.06396484375,
|
|
"learning_rate": 4.184725102275771e-07,
|
|
"loss": 6.410399510059506e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.54296875,
|
|
"reward_std": 0.2181890904903412,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1040,
|
|
"train_speed(iter/s)": 0.013199
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 508.0,
|
|
"completions/mean_length": 220.2265625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.6036532328211076,
|
|
"grad_norm": 0.466013362167736,
|
|
"kl": 0.0654296875,
|
|
"learning_rate": 4.183096155258201e-07,
|
|
"loss": 6.528668745886534e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.2181890904903412,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1041,
|
|
"train_speed(iter/s)": 0.013207
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 564.0,
|
|
"completions/mean_length": 232.546875,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.6042331110466802,
|
|
"grad_norm": 0.5403003282374381,
|
|
"kl": 0.065673828125,
|
|
"learning_rate": 4.18146594338049e-07,
|
|
"loss": 6.559432222275063e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62109375,
|
|
"reward_std": 0.2228916585445404,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1042,
|
|
"train_speed(iter/s)": 0.013215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 514.0,
|
|
"completions/mean_length": 245.046875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.6048129892722528,
|
|
"grad_norm": 0.5709313407777424,
|
|
"kl": 0.0673828125,
|
|
"learning_rate": 4.179834468081474e-07,
|
|
"loss": 6.749520252924412e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.31705188751220703,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 1043,
|
|
"train_speed(iter/s)": 0.013223
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 593.0,
|
|
"completions/mean_length": 231.078125,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.6053928674978255,
|
|
"grad_norm": 0.5357299764148787,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.1782017308011053e-07,
|
|
"loss": 6.938982551218942e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.23650971055030823,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1044,
|
|
"train_speed(iter/s)": 0.013232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9800.0,
|
|
"completions/mean_length": 327.5703125,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.6059727457233981,
|
|
"grad_norm": 0.4893098226274291,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 4.1765677329804473e-07,
|
|
"loss": 6.439364369725809e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.1597251147031784,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1045,
|
|
"train_speed(iter/s)": 0.013196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 816.0,
|
|
"completions/mean_length": 269.265625,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.6065526239489707,
|
|
"grad_norm": 0.5934810143929099,
|
|
"kl": 0.06201171875,
|
|
"learning_rate": 4.174932476061678e-07,
|
|
"loss": 6.192439468577504e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.32054588198661804,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.4921875,
|
|
"rewards/FMTORM/std": 0.062253449112176895,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1046,
|
|
"train_speed(iter/s)": 0.013203
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/mean_length": 284.9765625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.6071325021745434,
|
|
"grad_norm": 0.5035300244819517,
|
|
"kl": 0.0626220703125,
|
|
"learning_rate": 4.1732959614880856e-07,
|
|
"loss": 6.249136640690267e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.27352428436279297,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1047,
|
|
"train_speed(iter/s)": 0.01321
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 601.0,
|
|
"completions/mean_length": 224.5703125,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.607712380400116,
|
|
"grad_norm": 0.7275927572008132,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.1716581907040686e-07,
|
|
"loss": 6.924504123162478e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.3453049063682556,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1048,
|
|
"train_speed(iter/s)": 0.013218
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 597.0,
|
|
"completions/mean_length": 228.5703125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.6082922586256886,
|
|
"grad_norm": 0.680049778592276,
|
|
"kl": 0.062744140625,
|
|
"learning_rate": 4.1700191651551347e-07,
|
|
"loss": 6.28394482191652e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.3671424686908722,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1049,
|
|
"train_speed(iter/s)": 0.013226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 535.0,
|
|
"completions/mean_length": 219.65625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.6088721368512612,
|
|
"grad_norm": 0.5739906597439753,
|
|
"kl": 0.078369140625,
|
|
"learning_rate": 4.1683788862878986e-07,
|
|
"loss": 7.845491199987009e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.3142244815826416,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1050,
|
|
"train_speed(iter/s)": 0.013234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 523.0,
|
|
"completions/mean_length": 219.1328125,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6094520150768339,
|
|
"grad_norm": 0.6595812965610347,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 4.1667373555500816e-07,
|
|
"loss": 6.98485819157213e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.2746010422706604,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1051,
|
|
"train_speed(iter/s)": 0.013242
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 763.0,
|
|
"completions/mean_length": 234.359375,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.6100318933024065,
|
|
"grad_norm": 0.40798234405339967,
|
|
"kl": 0.0673828125,
|
|
"learning_rate": 4.165094574390508e-07,
|
|
"loss": 6.72823516651988e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.68359375,
|
|
"reward_std": 0.13509787619113922,
|
|
"rewards/CSTORM/mean": 0.38671875,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 1052,
|
|
"train_speed(iter/s)": 0.01325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 970.0,
|
|
"completions/mean_length": 269.796875,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.6106117715279791,
|
|
"grad_norm": 0.5748445374742622,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 4.16345054425911e-07,
|
|
"loss": 6.429508357541636e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.2458132952451706,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1053,
|
|
"train_speed(iter/s)": 0.013257
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 573.0,
|
|
"completions/mean_length": 250.609375,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.6111916497535518,
|
|
"grad_norm": 0.588545490725814,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 4.161805266606917e-07,
|
|
"loss": 6.771752669010311e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.3039947748184204,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1054,
|
|
"train_speed(iter/s)": 0.013264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 581.0,
|
|
"completions/mean_length": 223.4609375,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.6117715279791244,
|
|
"grad_norm": 0.5570749132110834,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 4.1601587428860626e-07,
|
|
"loss": 6.902914174133912e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.2402673065662384,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 1055,
|
|
"train_speed(iter/s)": 0.013273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 832.0,
|
|
"completions/mean_length": 258.984375,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.612351406204697,
|
|
"grad_norm": 0.5420954644245558,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 4.1585109745497805e-07,
|
|
"loss": 6.880619912408292e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.24887818098068237,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1056,
|
|
"train_speed(iter/s)": 0.01328
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 665.0,
|
|
"completions/mean_length": 272.0390625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.6129312844302697,
|
|
"grad_norm": 0.6096170454591935,
|
|
"kl": 0.069091796875,
|
|
"learning_rate": 4.1568619630524e-07,
|
|
"loss": 6.920294254086912e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.2949736714363098,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1057,
|
|
"train_speed(iter/s)": 0.013287
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 600.0,
|
|
"completions/mean_length": 247.984375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.6135111626558423,
|
|
"grad_norm": 0.5966163541967713,
|
|
"kl": 0.063232421875,
|
|
"learning_rate": 4.155211709849352e-07,
|
|
"loss": 6.313786434475332e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.2282869815826416,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1058,
|
|
"train_speed(iter/s)": 0.013295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 548.0,
|
|
"completions/mean_length": 210.828125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.6140910408814149,
|
|
"grad_norm": 0.5454863079724204,
|
|
"kl": 0.0771484375,
|
|
"learning_rate": 4.1535602163971597e-07,
|
|
"loss": 7.714932144153863e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.16755647957324982,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1059,
|
|
"train_speed(iter/s)": 0.013303
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 653.0,
|
|
"completions/mean_length": 247.1015625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.6146709191069876,
|
|
"grad_norm": 0.518505160572554,
|
|
"kl": 0.06298828125,
|
|
"learning_rate": 4.151907484153441e-07,
|
|
"loss": 6.288881559157744e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.18814769387245178,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1060,
|
|
"train_speed(iter/s)": 0.013311
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 627.0,
|
|
"completions/mean_length": 222.9453125,
|
|
"completions/min_length": 64.0,
|
|
"epoch": 0.6152507973325602,
|
|
"grad_norm": 0.5517616443845914,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.150253514576911e-07,
|
|
"loss": 6.929731171112508e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.23424318432807922,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1061,
|
|
"train_speed(iter/s)": 0.013318
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1131.0,
|
|
"completions/mean_length": 261.859375,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6158306755581328,
|
|
"grad_norm": 0.4930776670147069,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 4.1485983091273715e-07,
|
|
"loss": 6.644334644079208e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.24525238573551178,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1062,
|
|
"train_speed(iter/s)": 0.013324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 805.0,
|
|
"completions/mean_length": 250.4609375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.6164105537837055,
|
|
"grad_norm": 0.6090274401489454,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 4.1469418692657184e-07,
|
|
"loss": 6.985868094488978e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.3159300684928894,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1063,
|
|
"train_speed(iter/s)": 0.013331
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 566.0,
|
|
"completions/mean_length": 242.8203125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.6169904320092781,
|
|
"grad_norm": 0.4936243932039542,
|
|
"kl": 0.0654296875,
|
|
"learning_rate": 4.145284196453937e-07,
|
|
"loss": 6.554296123795211e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.17670938372612,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1064,
|
|
"train_speed(iter/s)": 0.013339
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 983.0,
|
|
"completions/mean_length": 230.203125,
|
|
"completions/min_length": 59.0,
|
|
"epoch": 0.6175703102348507,
|
|
"grad_norm": 0.5488179599140554,
|
|
"kl": 0.073974609375,
|
|
"learning_rate": 4.1436252921551014e-07,
|
|
"loss": 7.387969526462257e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.2350226789712906,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1065,
|
|
"train_speed(iter/s)": 0.013346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 556.0,
|
|
"completions/mean_length": 234.828125,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.6181501884604234,
|
|
"grad_norm": 0.5715715443642099,
|
|
"kl": 0.065185546875,
|
|
"learning_rate": 4.141965157833369e-07,
|
|
"loss": 6.503944314317778e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.2402673065662384,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1066,
|
|
"train_speed(iter/s)": 0.013353
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 957.0,
|
|
"completions/mean_length": 248.71875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.618730066685996,
|
|
"grad_norm": 0.5187191260995695,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 4.140303794953987e-07,
|
|
"loss": 6.642821244895458e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6953125,
|
|
"reward_std": 0.1652711033821106,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 1067,
|
|
"train_speed(iter/s)": 0.01336
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1077.0,
|
|
"completions/mean_length": 249.453125,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.6193099449115685,
|
|
"grad_norm": 0.5178129713796498,
|
|
"kl": 0.0654296875,
|
|
"learning_rate": 4.1386412049832853e-07,
|
|
"loss": 6.536793080158532e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.2824135422706604,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1068,
|
|
"train_speed(iter/s)": 0.013366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 710.0,
|
|
"completions/mean_length": 262.171875,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.6198898231371412,
|
|
"grad_norm": 0.5527431900012991,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 4.1369773893886763e-07,
|
|
"loss": 6.873454549349844e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.2373080551624298,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1069,
|
|
"train_speed(iter/s)": 0.013374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 904.0,
|
|
"completions/mean_length": 278.46875,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.6204697013627138,
|
|
"grad_norm": 0.4266844330569614,
|
|
"kl": 0.06298828125,
|
|
"learning_rate": 4.135312349638654e-07,
|
|
"loss": 6.29768765065819e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.203125,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1070,
|
|
"train_speed(iter/s)": 0.013368
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 577.0,
|
|
"completions/mean_length": 233.734375,
|
|
"completions/min_length": 111.0,
|
|
"epoch": 0.6210495795882864,
|
|
"grad_norm": 0.5967979086491316,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 4.133646087202795e-07,
|
|
"loss": 6.985636719036847e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2620859742164612,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1071,
|
|
"train_speed(iter/s)": 0.013375
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 910.0,
|
|
"completions/mean_length": 267.4375,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.6216294578138591,
|
|
"grad_norm": 0.552251794481405,
|
|
"kl": 0.05810546875,
|
|
"learning_rate": 4.131978603551753e-07,
|
|
"loss": 5.808494461234659e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.24525238573551178,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1072,
|
|
"train_speed(iter/s)": 0.013382
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 553.0,
|
|
"completions/mean_length": 236.7578125,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.6222093360394317,
|
|
"grad_norm": 0.5905629615918204,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 4.130309900157259e-07,
|
|
"loss": 7.011725392658263e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.24324540793895721,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1073,
|
|
"train_speed(iter/s)": 0.013389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 620.0,
|
|
"completions/mean_length": 238.1640625,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.6227892142650043,
|
|
"grad_norm": 0.5239474613063045,
|
|
"kl": 0.068603515625,
|
|
"learning_rate": 4.128639978492122e-07,
|
|
"loss": 6.867058982606977e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.20079457759857178,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1074,
|
|
"train_speed(iter/s)": 0.013397
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 551.0,
|
|
"completions/mean_length": 229.7734375,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6233690924905769,
|
|
"grad_norm": 0.5266611645720667,
|
|
"kl": 0.07080078125,
|
|
"learning_rate": 4.1269688400302275e-07,
|
|
"loss": 7.089480641297996e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66015625,
|
|
"reward_std": 0.24525238573551178,
|
|
"rewards/CSTORM/mean": 0.37109375,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1075,
|
|
"train_speed(iter/s)": 0.013404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 639.0,
|
|
"completions/mean_length": 270.40625,
|
|
"completions/min_length": 62.0,
|
|
"epoch": 0.6239489707161496,
|
|
"grad_norm": 0.5604943251570851,
|
|
"kl": 0.09619140625,
|
|
"learning_rate": 4.1252964862465323e-07,
|
|
"loss": 9.610097913537174e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.29640084505081177,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1076,
|
|
"train_speed(iter/s)": 0.013412
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 829.0,
|
|
"completions/mean_length": 266.3203125,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.6245288489417222,
|
|
"grad_norm": 0.5546845425177169,
|
|
"kl": 0.06396484375,
|
|
"learning_rate": 4.1236229186170665e-07,
|
|
"loss": 6.407064211089164e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.2919955849647522,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1077,
|
|
"train_speed(iter/s)": 0.013417
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1085.0,
|
|
"completions/mean_length": 257.90625,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.6251087271672948,
|
|
"grad_norm": 0.4932211547629552,
|
|
"kl": 0.066162109375,
|
|
"learning_rate": 4.121948138618932e-07,
|
|
"loss": 6.636283796979114e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.207808718085289,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1078,
|
|
"train_speed(iter/s)": 0.013423
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 695.0,
|
|
"completions/mean_length": 259.5546875,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.6256886053928675,
|
|
"grad_norm": 0.5945830917436976,
|
|
"kl": 0.064208984375,
|
|
"learning_rate": 4.1202721477303014e-07,
|
|
"loss": 6.416675023501739e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.3376619815826416,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1079,
|
|
"train_speed(iter/s)": 0.01343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 642.0,
|
|
"completions/mean_length": 236.390625,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.6262684836184401,
|
|
"grad_norm": 0.6294541895089102,
|
|
"kl": 0.068603515625,
|
|
"learning_rate": 4.118594947430415e-07,
|
|
"loss": 6.869600474601611e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.30371227860450745,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1080,
|
|
"train_speed(iter/s)": 0.013424
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1026.0,
|
|
"completions/mean_length": 256.8828125,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.6268483618440127,
|
|
"grad_norm": 0.5696952453286918,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 4.116916539199581e-07,
|
|
"loss": 6.998214666964486e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.57421875,
|
|
"reward_std": 0.2517244815826416,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1081,
|
|
"train_speed(iter/s)": 0.01343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 965.0,
|
|
"completions/mean_length": 279.5390625,
|
|
"completions/min_length": 116.0,
|
|
"epoch": 0.6274282400695854,
|
|
"grad_norm": 0.481234245764046,
|
|
"kl": 0.05517578125,
|
|
"learning_rate": 4.115236924519173e-07,
|
|
"loss": 5.5199157941387966e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.1338704228401184,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1082,
|
|
"train_speed(iter/s)": 0.013436
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 943.0,
|
|
"completions/mean_length": 245.6171875,
|
|
"completions/min_length": 59.0,
|
|
"epoch": 0.628008118295158,
|
|
"grad_norm": 0.5299945961205893,
|
|
"kl": 0.0712890625,
|
|
"learning_rate": 4.11355610487163e-07,
|
|
"loss": 7.137750799302012e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.18033519387245178,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1083,
|
|
"train_speed(iter/s)": 0.013441
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1221.0,
|
|
"completions/mean_length": 261.15625,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.6285879965207306,
|
|
"grad_norm": 0.5927966247162502,
|
|
"kl": 0.062744140625,
|
|
"learning_rate": 4.1118740817404556e-07,
|
|
"loss": 6.273837061598897e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.2572515904903412,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1084,
|
|
"train_speed(iter/s)": 0.013447
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 734.0,
|
|
"completions/mean_length": 257.1484375,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.6291678747463033,
|
|
"grad_norm": 0.33051144363376034,
|
|
"kl": 0.057861328125,
|
|
"learning_rate": 4.1101908566102134e-07,
|
|
"loss": 5.798170604975894e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.11584708839654922,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1085,
|
|
"train_speed(iter/s)": 0.013453
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 518.0,
|
|
"completions/mean_length": 251.484375,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.6297477529718759,
|
|
"grad_norm": 0.5420644113283827,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 4.1085064309665303e-07,
|
|
"loss": 6.592491990886629e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.2949736714363098,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1086,
|
|
"train_speed(iter/s)": 0.013461
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 598.0,
|
|
"completions/mean_length": 236.21875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.6303276311974485,
|
|
"grad_norm": 0.5582524251881138,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 4.10682080629609e-07,
|
|
"loss": 6.993173155933619e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6484375,
|
|
"reward_std": 0.2482304871082306,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 1087,
|
|
"train_speed(iter/s)": 0.013469
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 773.0,
|
|
"completions/mean_length": 264.6953125,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.6309075094230212,
|
|
"grad_norm": 0.6144292538678332,
|
|
"kl": 0.062255859375,
|
|
"learning_rate": 4.1051339840866373e-07,
|
|
"loss": 6.231776205822825e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.335225909948349,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 1088,
|
|
"train_speed(iter/s)": 0.013476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1273.0,
|
|
"completions/mean_length": 269.8046875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.6314873876485938,
|
|
"grad_norm": 0.48579335286413633,
|
|
"kl": 0.0673828125,
|
|
"learning_rate": 4.1034459658269715e-07,
|
|
"loss": 6.737866351613775e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.19746607542037964,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1089,
|
|
"train_speed(iter/s)": 0.013481
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 574.0,
|
|
"completions/mean_length": 239.3828125,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.6320672658741664,
|
|
"grad_norm": 0.46632701785712044,
|
|
"kl": 0.0634765625,
|
|
"learning_rate": 4.101756753006949e-07,
|
|
"loss": 6.33963500149548e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6328125,
|
|
"reward_std": 0.1791265904903412,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1090,
|
|
"train_speed(iter/s)": 0.013489
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1409.0,
|
|
"completions/mean_length": 305.078125,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.6326471440997391,
|
|
"grad_norm": 0.5881351660314701,
|
|
"kl": 0.0560302734375,
|
|
"learning_rate": 4.100066347117481e-07,
|
|
"loss": 5.594146205112338e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.2760012745857239,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1091,
|
|
"train_speed(iter/s)": 0.013493
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1231.0,
|
|
"completions/mean_length": 256.6171875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6332270223253117,
|
|
"grad_norm": 0.34833586463733396,
|
|
"kl": 0.067138671875,
|
|
"learning_rate": 4.09837474965053e-07,
|
|
"loss": 6.714269693475217e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.109375,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1092,
|
|
"train_speed(iter/s)": 0.013483
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 839.0,
|
|
"completions/mean_length": 266.28125,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.6338069005508843,
|
|
"grad_norm": 0.6553723330772019,
|
|
"kl": 0.0673828125,
|
|
"learning_rate": 4.096681962099112e-07,
|
|
"loss": 6.734758790116757e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.33737948536872864,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1093,
|
|
"train_speed(iter/s)": 0.01349
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 970.0,
|
|
"completions/mean_length": 245.390625,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.634386778776457,
|
|
"grad_norm": 0.5644769971576271,
|
|
"kl": 0.066650390625,
|
|
"learning_rate": 4.0949879859572917e-07,
|
|
"loss": 6.648338603554294e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.2492883950471878,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1094,
|
|
"train_speed(iter/s)": 0.013496
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 550.0,
|
|
"completions/mean_length": 235.109375,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.6349666570020296,
|
|
"grad_norm": 0.5458580843007298,
|
|
"kl": 0.066162109375,
|
|
"learning_rate": 4.0932928227201835e-07,
|
|
"loss": 6.599574408028275e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.1911257952451706,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1095,
|
|
"train_speed(iter/s)": 0.013504
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 766.0,
|
|
"completions/mean_length": 266.796875,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.6355465352276022,
|
|
"grad_norm": 0.5093001975523342,
|
|
"kl": 0.0604248046875,
|
|
"learning_rate": 4.091596473883951e-07,
|
|
"loss": 6.037508137524128e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.2427033632993698,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1096,
|
|
"train_speed(iter/s)": 0.013511
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1145.0,
|
|
"completions/mean_length": 254.5546875,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.6361264134531749,
|
|
"grad_norm": 0.5718017009411911,
|
|
"kl": 0.0673828125,
|
|
"learning_rate": 4.0898989409458007e-07,
|
|
"loss": 6.724029663018882e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2793486714363098,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1097,
|
|
"train_speed(iter/s)": 0.013517
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 922.0,
|
|
"completions/mean_length": 267.6484375,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.6367062916787475,
|
|
"grad_norm": 0.5130713223335258,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 4.088200225403988e-07,
|
|
"loss": 6.799183029215783e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.1868072748184204,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1098,
|
|
"train_speed(iter/s)": 0.013523
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 626.0,
|
|
"completions/mean_length": 220.7578125,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.6372861699043201,
|
|
"grad_norm": 0.5114287959350559,
|
|
"kl": 0.07373046875,
|
|
"learning_rate": 4.0865003287578104e-07,
|
|
"loss": 7.374790584435686e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.15894562005996704,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1099,
|
|
"train_speed(iter/s)": 0.013531
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 617.0,
|
|
"completions/mean_length": 221.40625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.6378660481298927,
|
|
"grad_norm": 0.6098874975253453,
|
|
"kl": 0.07080078125,
|
|
"learning_rate": 4.084799252507607e-07,
|
|
"loss": 7.08815932739526e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.299160361289978,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1100,
|
|
"train_speed(iter/s)": 0.013538
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/mean_length": 261.15625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.6384459263554654,
|
|
"grad_norm": 0.6705713667505113,
|
|
"kl": 0.065185546875,
|
|
"learning_rate": 4.083096998154761e-07,
|
|
"loss": 6.534038402605802e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.38719484210014343,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1101,
|
|
"train_speed(iter/s)": 0.01354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1037.0,
|
|
"completions/mean_length": 255.828125,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.639025804581038,
|
|
"grad_norm": 0.4900790588188697,
|
|
"kl": 0.071533203125,
|
|
"learning_rate": 4.08139356720169e-07,
|
|
"loss": 7.163002737797797e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.2380007803440094,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1102,
|
|
"train_speed(iter/s)": 0.013546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1073.0,
|
|
"completions/mean_length": 243.9609375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.6396056828066106,
|
|
"grad_norm": 0.6896230664112724,
|
|
"kl": 0.069091796875,
|
|
"learning_rate": 4.0796889611518567e-07,
|
|
"loss": 6.921983731444925e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.2871611714363098,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1103,
|
|
"train_speed(iter/s)": 0.013551
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 500.0,
|
|
"completions/mean_length": 230.2109375,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.6401855610321833,
|
|
"grad_norm": 0.5984933289398323,
|
|
"kl": 0.0732421875,
|
|
"learning_rate": 4.077983181509756e-07,
|
|
"loss": 7.323978934437037e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.61328125,
|
|
"reward_std": 0.2961822748184204,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1104,
|
|
"train_speed(iter/s)": 0.013559
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 596.0,
|
|
"completions/mean_length": 222.1015625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.6407654392577559,
|
|
"grad_norm": 0.5762158012091594,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 4.07627622978092e-07,
|
|
"loss": 6.673271127510816e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.16634787619113922,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1105,
|
|
"train_speed(iter/s)": 0.013566
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1017.0,
|
|
"completions/mean_length": 251.0859375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.6413453174833285,
|
|
"grad_norm": 0.6648649151500998,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 4.0745681074719163e-07,
|
|
"loss": 6.573605787707493e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.30319643020629883,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1106,
|
|
"train_speed(iter/s)": 0.013573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 680.0,
|
|
"completions/mean_length": 251.0390625,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.6419251957089012,
|
|
"grad_norm": 0.6626936816553374,
|
|
"kl": 0.064697265625,
|
|
"learning_rate": 4.072858816090346e-07,
|
|
"loss": 6.461357406806201e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.3010166883468628,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1107,
|
|
"train_speed(iter/s)": 0.01358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 852.0,
|
|
"completions/mean_length": 250.25,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.6425050739344738,
|
|
"grad_norm": 0.6065271269183818,
|
|
"kl": 0.069091796875,
|
|
"learning_rate": 4.071148357144838e-07,
|
|
"loss": 6.925190973561257e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.23828125,
|
|
"reward_std": 0.28375399112701416,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 1108,
|
|
"train_speed(iter/s)": 0.013587
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1062.0,
|
|
"completions/mean_length": 254.2109375,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.6430849521600464,
|
|
"grad_norm": 0.5224523671578893,
|
|
"kl": 0.064697265625,
|
|
"learning_rate": 4.0694367321450575e-07,
|
|
"loss": 6.477697752416134e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59765625,
|
|
"reward_std": 0.2180572748184204,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1109,
|
|
"train_speed(iter/s)": 0.013593
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1421.0,
|
|
"completions/mean_length": 252.78125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.6436648303856191,
|
|
"grad_norm": 0.5197170301174268,
|
|
"kl": 0.0760498046875,
|
|
"learning_rate": 4.0677239426016944e-07,
|
|
"loss": 7.58495443733409e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2690907418727875,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1110,
|
|
"train_speed(iter/s)": 0.013598
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1000.0,
|
|
"completions/mean_length": 261.171875,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.6442447086111917,
|
|
"grad_norm": 0.5283250922230726,
|
|
"kl": 0.064697265625,
|
|
"learning_rate": 4.0660099900264695e-07,
|
|
"loss": 6.480507727246732e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.2584601938724518,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1111,
|
|
"train_speed(iter/s)": 0.013602
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 609.0,
|
|
"completions/mean_length": 251.515625,
|
|
"completions/min_length": 124.0,
|
|
"epoch": 0.6448245868367642,
|
|
"grad_norm": 0.5330490205639803,
|
|
"kl": 0.0621337890625,
|
|
"learning_rate": 4.064294875932127e-07,
|
|
"loss": 6.218014459591359e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.27557218074798584,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1112,
|
|
"train_speed(iter/s)": 0.013609
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 652.0,
|
|
"completions/mean_length": 244.875,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.645404465062337,
|
|
"grad_norm": 0.6214183739274506,
|
|
"kl": 0.07080078125,
|
|
"learning_rate": 4.062578601832439e-07,
|
|
"loss": 7.083611853886396e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.221683070063591,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1113,
|
|
"train_speed(iter/s)": 0.013617
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 897.0,
|
|
"completions/mean_length": 278.3828125,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.6459843432879095,
|
|
"grad_norm": 0.5219617581173517,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 4.060861169242201e-07,
|
|
"loss": 6.62652455503121e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.17657756805419922,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1114,
|
|
"train_speed(iter/s)": 0.013624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 983.0,
|
|
"completions/mean_length": 286.9609375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.6465642215134821,
|
|
"grad_norm": 0.5053402003651721,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 4.0591425796772275e-07,
|
|
"loss": 7.054574234643951e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.21158519387245178,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 1115,
|
|
"train_speed(iter/s)": 0.013573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 962.0,
|
|
"completions/mean_length": 236.8359375,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.6471440997390548,
|
|
"grad_norm": 0.5508315921762471,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 4.057422834654361e-07,
|
|
"loss": 6.778124225093052e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.2300376147031784,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1116,
|
|
"train_speed(iter/s)": 0.01358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 836.0,
|
|
"completions/mean_length": 228.125,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.6477239779646274,
|
|
"grad_norm": 0.5352268595756057,
|
|
"kl": 0.072021484375,
|
|
"learning_rate": 4.055701935691457e-07,
|
|
"loss": 7.204852590803057e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.24013549089431763,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1117,
|
|
"train_speed(iter/s)": 0.013587
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 731.0,
|
|
"completions/mean_length": 240.1796875,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.6483038561902,
|
|
"grad_norm": 0.39658550261776876,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 4.053979884307393e-07,
|
|
"loss": 7.053598528727889e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.10639689117670059,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1118,
|
|
"train_speed(iter/s)": 0.013593
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1343.0,
|
|
"completions/mean_length": 308.2578125,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.6488837344157727,
|
|
"grad_norm": 0.6137395089726311,
|
|
"kl": 0.063720703125,
|
|
"learning_rate": 4.052256682022064e-07,
|
|
"loss": 6.372870120685548e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.41673195362091064,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1119,
|
|
"train_speed(iter/s)": 0.013598
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1132.0,
|
|
"completions/mean_length": 278.2421875,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.6494636126413453,
|
|
"grad_norm": 0.5789716956812467,
|
|
"kl": 0.05908203125,
|
|
"learning_rate": 4.0505323303563777e-07,
|
|
"loss": 5.9072437579743564e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.3622891902923584,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1120,
|
|
"train_speed(iter/s)": 0.013603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 583.0,
|
|
"completions/mean_length": 249.3828125,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.6500434908669179,
|
|
"grad_norm": 0.6218330037181012,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 4.04880683083226e-07,
|
|
"loss": 6.455255788750947e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.34987977147102356,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1121,
|
|
"train_speed(iter/s)": 0.013611
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 572.0,
|
|
"completions/mean_length": 238.4453125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.6506233690924905,
|
|
"grad_norm": 0.5162884260125271,
|
|
"kl": 0.210693359375,
|
|
"learning_rate": 4.047080184972647e-07,
|
|
"loss": 0.00020986884192097932,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.19341117143630981,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1122,
|
|
"train_speed(iter/s)": 0.013618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 517.0,
|
|
"completions/mean_length": 230.359375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.6512032473180632,
|
|
"grad_norm": 0.5766049468980169,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 4.045352394301489e-07,
|
|
"loss": 6.874749669805169e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.26238328218460083,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1123,
|
|
"train_speed(iter/s)": 0.013624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 650.0,
|
|
"completions/mean_length": 252.75,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.6517831255436358,
|
|
"grad_norm": 0.7543068120581412,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 4.0436234603437444e-07,
|
|
"loss": 7.520320650655776e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.4699135720729828,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1124,
|
|
"train_speed(iter/s)": 0.013631
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1088.0,
|
|
"completions/mean_length": 254.15625,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.6523630037692084,
|
|
"grad_norm": 0.5365204565176231,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 4.0418933846253817e-07,
|
|
"loss": 6.779567047487944e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.18546685576438904,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1125,
|
|
"train_speed(iter/s)": 0.013636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 987.0,
|
|
"completions/mean_length": 281.328125,
|
|
"completions/min_length": 122.0,
|
|
"epoch": 0.6529428819947811,
|
|
"grad_norm": 0.5945961701321668,
|
|
"kl": 0.069091796875,
|
|
"learning_rate": 4.040162168673378e-07,
|
|
"loss": 6.898510036990047e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.2991603910923004,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1126,
|
|
"train_speed(iter/s)": 0.013643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 555.0,
|
|
"completions/mean_length": 244.8203125,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.6535227602203537,
|
|
"grad_norm": 0.4690675523809889,
|
|
"kl": 0.0640869140625,
|
|
"learning_rate": 4.038429814015715e-07,
|
|
"loss": 6.40400976408273e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.16591878235340118,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1127,
|
|
"train_speed(iter/s)": 0.013649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 686.0,
|
|
"completions/mean_length": 242.546875,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.6541026384459263,
|
|
"grad_norm": 0.5128738970968477,
|
|
"kl": 0.074951171875,
|
|
"learning_rate": 4.0366963221813814e-07,
|
|
"loss": 7.503988308599219e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.61328125,
|
|
"reward_std": 0.23113328218460083,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1128,
|
|
"train_speed(iter/s)": 0.013656
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 587.0,
|
|
"completions/mean_length": 228.859375,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.654682516671499,
|
|
"grad_norm": 0.604142777103051,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 4.0349616947003685e-07,
|
|
"loss": 6.934395059943199e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.296875,
|
|
"reward_std": 0.30300477147102356,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5390625,
|
|
"rewards/VQAORM/std": 0.5004304051399231,
|
|
"step": 1129,
|
|
"train_speed(iter/s)": 0.013664
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 756.0,
|
|
"completions/mean_length": 286.453125,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.6552623948970716,
|
|
"grad_norm": 0.6143469818983888,
|
|
"kl": 0.05712890625,
|
|
"learning_rate": 4.0332259331036687e-07,
|
|
"loss": 5.7075973018072546e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.2691001296043396,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 1130,
|
|
"train_speed(iter/s)": 0.013671
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 572.0,
|
|
"completions/mean_length": 236.328125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.6558422731226442,
|
|
"grad_norm": 0.5381297296295514,
|
|
"kl": 0.156005859375,
|
|
"learning_rate": 4.031489038923278e-07,
|
|
"loss": 0.0001564046397106722,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.22664928436279297,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1131,
|
|
"train_speed(iter/s)": 0.013678
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 942.0,
|
|
"completions/mean_length": 262.7578125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.6564221513482169,
|
|
"grad_norm": 0.5718005198314157,
|
|
"kl": 0.065673828125,
|
|
"learning_rate": 4.02975101369219e-07,
|
|
"loss": 6.58957360428758e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.15234375,
|
|
"reward_std": 0.2600978910923004,
|
|
"rewards/CSTORM/mean": 0.20703125,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.4453125,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 1132,
|
|
"train_speed(iter/s)": 0.013683
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 555.0,
|
|
"completions/mean_length": 240.5625,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.6570020295737895,
|
|
"grad_norm": 0.6192667336810743,
|
|
"kl": 0.071044921875,
|
|
"learning_rate": 4.0280118589443973e-07,
|
|
"loss": 7.11945176590234e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.20391929149627686,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1133,
|
|
"train_speed(iter/s)": 0.013691
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1347.0,
|
|
"completions/mean_length": 297.3671875,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.6575819077993621,
|
|
"grad_norm": 1.4245335358547146,
|
|
"kl": 8.53173828125,
|
|
"learning_rate": 4.0262715762148905e-07,
|
|
"loss": 0.008544540032744408,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.1015625,
|
|
"reward_std": 0.3272815942764282,
|
|
"rewards/CSTORM/mean": 0.18359375,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.421875,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 1134,
|
|
"train_speed(iter/s)": 0.013629
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1671.0,
|
|
"completions/mean_length": 303.390625,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.6581617860249348,
|
|
"grad_norm": 9.426456126937271,
|
|
"kl": 0.0665283203125,
|
|
"learning_rate": 4.0245301670396526e-07,
|
|
"loss": 6.647142436122522e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.25873860716819763,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1135,
|
|
"train_speed(iter/s)": 0.013565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 608.0,
|
|
"completions/mean_length": 234.1640625,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6587416642505074,
|
|
"grad_norm": 0.6070054305106469,
|
|
"kl": 0.07763671875,
|
|
"learning_rate": 4.022787632955664e-07,
|
|
"loss": 7.756338163744658e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.2380007952451706,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1136,
|
|
"train_speed(iter/s)": 0.013573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 821.0,
|
|
"completions/mean_length": 240.0,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.65932154247608,
|
|
"grad_norm": 0.6466315667925993,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 4.0210439755008966e-07,
|
|
"loss": 6.988745008129627e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.3233773708343506,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1137,
|
|
"train_speed(iter/s)": 0.01358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 598.0,
|
|
"completions/mean_length": 233.6328125,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.6599014207016527,
|
|
"grad_norm": 0.5497369754303492,
|
|
"kl": 0.061767578125,
|
|
"learning_rate": 4.0192991962143146e-07,
|
|
"loss": 6.177488103276119e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.2529330849647522,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1138,
|
|
"train_speed(iter/s)": 0.013586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1173.0,
|
|
"completions/mean_length": 271.3828125,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.6604812989272253,
|
|
"grad_norm": 0.5873528643151381,
|
|
"kl": 0.09130859375,
|
|
"learning_rate": 4.0175532966358707e-07,
|
|
"loss": 9.1107256594114e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.3245859742164612,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1139,
|
|
"train_speed(iter/s)": 0.013537
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 614.0,
|
|
"completions/mean_length": 239.8515625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.6610611771527979,
|
|
"grad_norm": 0.5395915643404781,
|
|
"kl": 0.066162109375,
|
|
"learning_rate": 4.015806278306509e-07,
|
|
"loss": 6.624009984079748e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.23650971055030823,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1140,
|
|
"train_speed(iter/s)": 0.013543
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 674.0,
|
|
"completions/mean_length": 233.828125,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.6616410553783706,
|
|
"grad_norm": 0.6518461084903224,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 4.0140581427681577e-07,
|
|
"loss": 6.813518848503008e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.73828125,
|
|
"reward_std": 0.27559107542037964,
|
|
"rewards/CSTORM/mean": 0.38671875,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8515625,
|
|
"rewards/VQAORM/std": 0.356930136680603,
|
|
"step": 1141,
|
|
"train_speed(iter/s)": 0.01355
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 507.0,
|
|
"completions/mean_length": 219.859375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.6622209336039432,
|
|
"grad_norm": 0.6102057470792642,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 4.0123088915637334e-07,
|
|
"loss": 7.502907101297751e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.2793486714363098,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1142,
|
|
"train_speed(iter/s)": 0.013546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 715.0,
|
|
"completions/mean_length": 236.8828125,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6628008118295158,
|
|
"grad_norm": 0.5258885744424939,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 4.010558526237136e-07,
|
|
"loss": 6.637293699895963e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.202413409948349,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1143,
|
|
"train_speed(iter/s)": 0.013553
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1224.0,
|
|
"completions/mean_length": 280.34375,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.6633806900550885,
|
|
"grad_norm": 0.4446104238070893,
|
|
"kl": 0.0623779296875,
|
|
"learning_rate": 4.0088070483332513e-07,
|
|
"loss": 6.219666101969779e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.63671875,
|
|
"reward_std": 0.1800883561372757,
|
|
"rewards/CSTORM/mean": 0.375,
|
|
"rewards/CSTORM/std": 0.2173570692539215,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1144,
|
|
"train_speed(iter/s)": 0.013558
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 641.0,
|
|
"completions/mean_length": 231.8828125,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.6639605682806611,
|
|
"grad_norm": 0.49998113728680743,
|
|
"kl": 0.068359375,
|
|
"learning_rate": 4.007054459397944e-07,
|
|
"loss": 6.8489825935103e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.20200318098068237,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1145,
|
|
"train_speed(iter/s)": 0.013565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 486.0,
|
|
"completions/mean_length": 235.0546875,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.6645404465062337,
|
|
"grad_norm": 0.4562394287198903,
|
|
"kl": 0.0614013671875,
|
|
"learning_rate": 4.0053007609780616e-07,
|
|
"loss": 6.133799615781754e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.1362876147031784,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1146,
|
|
"train_speed(iter/s)": 0.013572
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 882.0,
|
|
"completions/mean_length": 254.7421875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.6651203247318063,
|
|
"grad_norm": 0.6575919502493632,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 4.003545954621428e-07,
|
|
"loss": 6.772854248993099e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.29296875,
|
|
"reward_std": 0.2817470133304596,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 1147,
|
|
"train_speed(iter/s)": 0.013579
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 586.0,
|
|
"completions/mean_length": 249.9765625,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.665700202957379,
|
|
"grad_norm": 0.591466479851208,
|
|
"kl": 0.0621337890625,
|
|
"learning_rate": 4.0017900418768505e-07,
|
|
"loss": 6.202429358381778e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.3268713653087616,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1148,
|
|
"train_speed(iter/s)": 0.013586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 657.0,
|
|
"completions/mean_length": 229.9765625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.6662800811829516,
|
|
"grad_norm": 0.5408915951887816,
|
|
"kl": 0.06396484375,
|
|
"learning_rate": 4.000033024294105e-07,
|
|
"loss": 6.3977568061091e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.22763928771018982,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1149,
|
|
"train_speed(iter/s)": 0.013594
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 906.0,
|
|
"completions/mean_length": 241.2421875,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.6668599594085242,
|
|
"grad_norm": 0.6605336486329257,
|
|
"kl": 0.069580078125,
|
|
"learning_rate": 3.9982749034239486e-07,
|
|
"loss": 6.97593204677105e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.4188928008079529,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1150,
|
|
"train_speed(iter/s)": 0.0136
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 612.0,
|
|
"completions/mean_length": 248.75,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.6674398376340969,
|
|
"grad_norm": 0.44976456452871494,
|
|
"kl": 0.063232421875,
|
|
"learning_rate": 3.996515680818112e-07,
|
|
"loss": 6.322664557956159e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.75390625,
|
|
"reward_std": 0.20903617143630981,
|
|
"rewards/CSTORM/mean": 0.41015625,
|
|
"rewards/CSTORM/std": 0.1927177608013153,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.84375,
|
|
"rewards/VQAORM/std": 0.3645188808441162,
|
|
"step": 1151,
|
|
"train_speed(iter/s)": 0.013607
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 511.0,
|
|
"completions/mean_length": 218.2421875,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6680197158596695,
|
|
"grad_norm": 0.5554753708130913,
|
|
"kl": 0.077392578125,
|
|
"learning_rate": 3.9947553580292945e-07,
|
|
"loss": 7.735656981822103e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1152,
|
|
"train_speed(iter/s)": 0.013615
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 769.0,
|
|
"completions/mean_length": 280.734375,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.668599594085242,
|
|
"grad_norm": 0.5188624140999841,
|
|
"kl": 0.0596923828125,
|
|
"learning_rate": 3.9929939366111705e-07,
|
|
"loss": 5.9668804169632494e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.3376619815826416,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1153,
|
|
"train_speed(iter/s)": 0.013609
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 615.0,
|
|
"completions/mean_length": 240.8359375,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.6691794723108148,
|
|
"grad_norm": 0.43452293152780136,
|
|
"kl": 0.068359375,
|
|
"learning_rate": 3.99123141811838e-07,
|
|
"loss": 6.816301902290434e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.71875,
|
|
"reward_std": 0.15853539109230042,
|
|
"rewards/CSTORM/mean": 0.3984375,
|
|
"rewards/CSTORM/std": 0.20195281505584717,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8203125,
|
|
"rewards/VQAORM/std": 0.3854355216026306,
|
|
"step": 1154,
|
|
"train_speed(iter/s)": 0.013617
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9801.0,
|
|
"completions/mean_length": 339.7109375,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.6697593505363874,
|
|
"grad_norm": 0.5232616180211368,
|
|
"kl": 0.058349609375,
|
|
"learning_rate": 3.9894678041065357e-07,
|
|
"loss": 5.836346099385992e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.2467750608921051,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1155,
|
|
"train_speed(iter/s)": 0.013584
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1448.0,
|
|
"completions/mean_length": 272.7734375,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.67033922876196,
|
|
"grad_norm": 5.826497870188806,
|
|
"kl": 0.132568359375,
|
|
"learning_rate": 3.9877030961322123e-07,
|
|
"loss": 0.00013223077985458076,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.23310349881649017,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.48828125,
|
|
"rewards/FMTORM/std": 0.07594143599271774,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1156,
|
|
"train_speed(iter/s)": 0.013523
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 654.0,
|
|
"completions/mean_length": 256.1953125,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.6709191069875327,
|
|
"grad_norm": 0.5798928552341635,
|
|
"kl": 0.0606689453125,
|
|
"learning_rate": 3.985937295752955e-07,
|
|
"loss": 6.067585127311759e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.2885015904903412,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1157,
|
|
"train_speed(iter/s)": 0.013531
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 539.0,
|
|
"completions/mean_length": 232.0859375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.6714989852131052,
|
|
"grad_norm": 0.5701541227999235,
|
|
"kl": 0.076416015625,
|
|
"learning_rate": 3.9841704045272704e-07,
|
|
"loss": 7.637096859980375e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.2559111714363098,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1158,
|
|
"train_speed(iter/s)": 0.013538
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 688.0,
|
|
"completions/mean_length": 272.6953125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.6720788634386778,
|
|
"grad_norm": 0.6549463068352133,
|
|
"kl": 0.063232421875,
|
|
"learning_rate": 3.9824024240146294e-07,
|
|
"loss": 6.331436452455819e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.3495104908943176,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1159,
|
|
"train_speed(iter/s)": 0.013545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 740.0,
|
|
"completions/mean_length": 244.484375,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.6726587416642505,
|
|
"grad_norm": 0.6246889200456196,
|
|
"kl": 0.0712890625,
|
|
"learning_rate": 3.9806333557754606e-07,
|
|
"loss": 7.107699639163911e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.29807955026626587,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1160,
|
|
"train_speed(iter/s)": 0.013552
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1267.0,
|
|
"completions/mean_length": 279.3828125,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.6732386198898231,
|
|
"grad_norm": 0.4580964265879429,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 3.978863201371157e-07,
|
|
"loss": 6.446604675147682e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.19596019387245178,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1161,
|
|
"train_speed(iter/s)": 0.013556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 881.0,
|
|
"completions/mean_length": 283.1171875,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.6738184981153957,
|
|
"grad_norm": 0.5084364436546095,
|
|
"kl": 0.0565185546875,
|
|
"learning_rate": 3.9770919623640687e-07,
|
|
"loss": 5.647566285915673e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.231912761926651,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 1162,
|
|
"train_speed(iter/s)": 0.013563
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 889.0,
|
|
"completions/mean_length": 263.3515625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.6743983763409684,
|
|
"grad_norm": 0.4703372092711439,
|
|
"kl": 0.063720703125,
|
|
"learning_rate": 3.975319640317502e-07,
|
|
"loss": 6.362138083204627e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.22750747203826904,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1163,
|
|
"train_speed(iter/s)": 0.013569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 544.0,
|
|
"completions/mean_length": 251.7734375,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.674978254566541,
|
|
"grad_norm": 0.5705193935807465,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 3.973546236795722e-07,
|
|
"loss": 6.935890996828675e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.2572515904903412,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 1164,
|
|
"train_speed(iter/s)": 0.013576
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1268.0,
|
|
"completions/mean_length": 255.7734375,
|
|
"completions/min_length": 121.0,
|
|
"epoch": 0.6755581327921136,
|
|
"grad_norm": 0.5509105838292069,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 3.971771753363944e-07,
|
|
"loss": 6.691671296721324e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.26989442110061646,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1165,
|
|
"train_speed(iter/s)": 0.013581
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 482.0,
|
|
"completions/mean_length": 210.84375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.6761380110176863,
|
|
"grad_norm": 0.4297111019792125,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 3.9699961915883394e-07,
|
|
"loss": 7.525105320382863e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.1682041585445404,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1166,
|
|
"train_speed(iter/s)": 0.013588
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 759.0,
|
|
"completions/mean_length": 263.03125,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.6767178892432589,
|
|
"grad_norm": 0.6010186177746494,
|
|
"kl": 0.068603515625,
|
|
"learning_rate": 3.9682195530360326e-07,
|
|
"loss": 6.852919614175335e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.26963484287261963,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1167,
|
|
"train_speed(iter/s)": 0.013595
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 696.0,
|
|
"completions/mean_length": 238.5078125,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.6772977674688315,
|
|
"grad_norm": 0.4723336405184464,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 3.9664418392750946e-07,
|
|
"loss": 6.806164310546592e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.19056488573551178,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1168,
|
|
"train_speed(iter/s)": 0.013602
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1014.0,
|
|
"completions/mean_length": 241.7890625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.6778776456944042,
|
|
"grad_norm": 0.578090949152948,
|
|
"kl": 0.064208984375,
|
|
"learning_rate": 3.964663051874548e-07,
|
|
"loss": 6.42515515210107e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.2715361714363098,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1169,
|
|
"train_speed(iter/s)": 0.013606
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 842.0,
|
|
"completions/mean_length": 281.4609375,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.6784575239199768,
|
|
"grad_norm": 0.5026415558864903,
|
|
"kl": 0.0697021484375,
|
|
"learning_rate": 3.962883192404362e-07,
|
|
"loss": 6.958463927730918e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.24969863891601562,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1170,
|
|
"train_speed(iter/s)": 0.013612
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 590.0,
|
|
"completions/mean_length": 233.9609375,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.6790374021455494,
|
|
"grad_norm": 0.30751889074433714,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 3.9611022624354526e-07,
|
|
"loss": 6.926356581971049e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.11045178771018982,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1171,
|
|
"train_speed(iter/s)": 0.01362
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 960.0,
|
|
"completions/mean_length": 236.0078125,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.679617280371122,
|
|
"grad_norm": 0.6549109140163499,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 3.95932026353968e-07,
|
|
"loss": 6.789962208131328e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.30477428436279297,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1172,
|
|
"train_speed(iter/s)": 0.013614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 518.0,
|
|
"completions/mean_length": 256.15625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.6801971585966947,
|
|
"grad_norm": 0.5523274410947396,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 3.957537197289849e-07,
|
|
"loss": 6.999612378422171e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.20836961269378662,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1173,
|
|
"train_speed(iter/s)": 0.013621
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 736.0,
|
|
"completions/mean_length": 246.6015625,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.6807770368222673,
|
|
"grad_norm": 0.5285946567207657,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 3.955753065259705e-07,
|
|
"loss": 6.823844159953296e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.2578125,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 1174,
|
|
"train_speed(iter/s)": 0.013628
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 875.0,
|
|
"completions/mean_length": 249.65625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.6813569150478399,
|
|
"grad_norm": 0.45676336444212823,
|
|
"kl": 0.071044921875,
|
|
"learning_rate": 3.953967869023935e-07,
|
|
"loss": 7.105170516297221e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.16634787619113922,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1175,
|
|
"train_speed(iter/s)": 0.013634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 671.0,
|
|
"completions/mean_length": 244.6328125,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6819367932734126,
|
|
"grad_norm": 0.5122702658888054,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 3.952181610158164e-07,
|
|
"loss": 6.816616951255128e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.3041265904903412,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1176,
|
|
"train_speed(iter/s)": 0.013641
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9859.0,
|
|
"completions/mean_length": 336.921875,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.6825166714989852,
|
|
"grad_norm": 0.4456415997160101,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 3.950394290238958e-07,
|
|
"loss": 6.629458221141249e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.218038409948349,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1177,
|
|
"train_speed(iter/s)": 0.013607
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 631.0,
|
|
"completions/mean_length": 245.8828125,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.6830965497245578,
|
|
"grad_norm": 0.5129499233844925,
|
|
"kl": 0.067138671875,
|
|
"learning_rate": 3.9486059108438153e-07,
|
|
"loss": 6.704492989229038e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.24205568432807922,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1178,
|
|
"train_speed(iter/s)": 0.013613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 834.0,
|
|
"completions/mean_length": 268.8828125,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.6836764279501305,
|
|
"grad_norm": 0.5529398073449829,
|
|
"kl": 0.0625,
|
|
"learning_rate": 3.9468164735511727e-07,
|
|
"loss": 6.246186967473477e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.32421875,
|
|
"reward_std": 0.272725909948349,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 1179,
|
|
"train_speed(iter/s)": 0.013618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 998.0,
|
|
"completions/mean_length": 283.53125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.6842563061757031,
|
|
"grad_norm": 0.44554983868861,
|
|
"kl": 0.063720703125,
|
|
"learning_rate": 3.945025979940399e-07,
|
|
"loss": 6.372816278599203e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.18775954842567444,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1180,
|
|
"train_speed(iter/s)": 0.013623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 838.0,
|
|
"completions/mean_length": 286.140625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.6848361844012757,
|
|
"grad_norm": 0.4377044046628408,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 3.9432344315917956e-07,
|
|
"loss": 6.642797234235331e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.1833132952451706,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1181,
|
|
"train_speed(iter/s)": 0.01363
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 775.0,
|
|
"completions/mean_length": 247.65625,
|
|
"completions/min_length": 118.0,
|
|
"epoch": 0.6854160626268484,
|
|
"grad_norm": 0.5945290309872817,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 3.9414418300865947e-07,
|
|
"loss": 6.791854684706777e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3125,
|
|
"reward_std": 0.23890803754329681,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 1182,
|
|
"train_speed(iter/s)": 0.013636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 711.0,
|
|
"completions/mean_length": 251.09375,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.685995940852421,
|
|
"grad_norm": 0.5049956436295838,
|
|
"kl": 0.072265625,
|
|
"learning_rate": 3.9396481770069585e-07,
|
|
"loss": 7.217950042104349e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.25940513610839844,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1183,
|
|
"train_speed(iter/s)": 0.013642
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 965.0,
|
|
"completions/mean_length": 255.375,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.6865758190779936,
|
|
"grad_norm": 0.5415807243119399,
|
|
"kl": 0.063720703125,
|
|
"learning_rate": 3.937853473935977e-07,
|
|
"loss": 6.355570076266304e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.29156649112701416,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1184,
|
|
"train_speed(iter/s)": 0.013648
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 928.0,
|
|
"completions/mean_length": 295.5,
|
|
"completions/min_length": 66.0,
|
|
"epoch": 0.6871556973035663,
|
|
"grad_norm": 0.5959621751465165,
|
|
"kl": 0.063232421875,
|
|
"learning_rate": 3.9360577224576667e-07,
|
|
"loss": 6.325166032183915e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.322579026222229,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1185,
|
|
"train_speed(iter/s)": 0.013655
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 697.0,
|
|
"completions/mean_length": 232.984375,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.6877355755291389,
|
|
"grad_norm": 0.5223242481218353,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 3.934260924156971e-07,
|
|
"loss": 7.515265315305442e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1186,
|
|
"train_speed(iter/s)": 0.013662
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 789.0,
|
|
"completions/mean_length": 268.09375,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.6883154537547115,
|
|
"grad_norm": 0.582537738857288,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 3.932463080619754e-07,
|
|
"loss": 6.748024316038936e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.27715006470680237,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1187,
|
|
"train_speed(iter/s)": 0.013668
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 734.0,
|
|
"completions/mean_length": 285.8515625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.6888953319802842,
|
|
"grad_norm": 0.44727823183428606,
|
|
"kl": 0.06103515625,
|
|
"learning_rate": 3.9306641934328053e-07,
|
|
"loss": 6.115720316302031e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.2312650829553604,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1188,
|
|
"train_speed(iter/s)": 0.013675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 664.0,
|
|
"completions/mean_length": 256.3046875,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.6894752102058568,
|
|
"grad_norm": 0.5807005346867037,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 3.928864264183835e-07,
|
|
"loss": 7.011223351582885e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.2686898708343506,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 1189,
|
|
"train_speed(iter/s)": 0.013682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 659.0,
|
|
"completions/mean_length": 261.0390625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.6900550884314294,
|
|
"grad_norm": 0.5121674956572306,
|
|
"kl": 0.147705078125,
|
|
"learning_rate": 3.927063294461472e-07,
|
|
"loss": 0.00014735865988768637,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.20703125,
|
|
"reward_std": 0.15732678771018982,
|
|
"rewards/CSTORM/mean": 0.21875,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.4921875,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 1190,
|
|
"train_speed(iter/s)": 0.013688
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 937.0,
|
|
"completions/mean_length": 267.5078125,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.6906349666570021,
|
|
"grad_norm": 0.5726277863261198,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 3.9252612858552647e-07,
|
|
"loss": 6.622055661864579e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.23864847421646118,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1191,
|
|
"train_speed(iter/s)": 0.013694
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 519.0,
|
|
"completions/mean_length": 220.421875,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.6912148448825747,
|
|
"grad_norm": 0.5569870541378024,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.923458239955677e-07,
|
|
"loss": 7.299685967154801e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.20001506805419922,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1192,
|
|
"train_speed(iter/s)": 0.013701
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 586.0,
|
|
"completions/mean_length": 252.0078125,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.6917947231081473,
|
|
"grad_norm": 0.48300764344699465,
|
|
"kl": 0.0615234375,
|
|
"learning_rate": 3.9216541583540894e-07,
|
|
"loss": 6.152774585643783e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.12888534367084503,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1193,
|
|
"train_speed(iter/s)": 0.013708
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 555.0,
|
|
"completions/mean_length": 244.8984375,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.6923746013337199,
|
|
"grad_norm": 0.5809456849265694,
|
|
"kl": 0.134521484375,
|
|
"learning_rate": 3.919849042642797e-07,
|
|
"loss": 0.00013484721421264112,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.3344370424747467,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1194,
|
|
"train_speed(iter/s)": 0.013702
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 900.0,
|
|
"completions/mean_length": 249.8046875,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.6929544795592926,
|
|
"grad_norm": 0.45527213497516594,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 3.9180428944150057e-07,
|
|
"loss": 6.600260530831292e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.188015878200531,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1195,
|
|
"train_speed(iter/s)": 0.013708
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 793.0,
|
|
"completions/mean_length": 253.6875,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.6935343577848652,
|
|
"grad_norm": 115.78946777061641,
|
|
"kl": 172.03564453125,
|
|
"learning_rate": 3.9162357152648343e-07,
|
|
"loss": 0.17194244265556335,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66796875,
|
|
"reward_std": 0.3211236000061035,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1196,
|
|
"train_speed(iter/s)": 0.013649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 694.0,
|
|
"completions/mean_length": 253.5546875,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.6941142360104378,
|
|
"grad_norm": 0.4073410234203987,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 3.9144275067873114e-07,
|
|
"loss": 6.787678285036236e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.1478765904903412,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1197,
|
|
"train_speed(iter/s)": 0.013655
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 759.0,
|
|
"completions/mean_length": 249.078125,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.6946941142360105,
|
|
"grad_norm": 0.5308963157392763,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 3.9126182705783735e-07,
|
|
"loss": 6.773817585781217e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.250515878200531,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1198,
|
|
"train_speed(iter/s)": 0.013662
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 610.0,
|
|
"completions/mean_length": 246.8671875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.6952739924615831,
|
|
"grad_norm": 0.5841923117094203,
|
|
"kl": 0.062744140625,
|
|
"learning_rate": 3.9108080082348635e-07,
|
|
"loss": 6.28580164629966e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.3843863010406494,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1199,
|
|
"train_speed(iter/s)": 0.013669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 857.0,
|
|
"completions/mean_length": 265.734375,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.6958538706871557,
|
|
"grad_norm": 0.567309055319741,
|
|
"kl": 0.066650390625,
|
|
"learning_rate": 3.908996721354533e-07,
|
|
"loss": 6.671289884252474e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.765625,
|
|
"reward_std": 0.20541039109230042,
|
|
"rewards/CSTORM/mean": 0.40625,
|
|
"rewards/CSTORM/std": 0.19592301547527313,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.859375,
|
|
"rewards/VQAORM/std": 0.3490002751350403,
|
|
"step": 1200,
|
|
"train_speed(iter/s)": 0.013676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1269.0,
|
|
"completions/mean_length": 257.4609375,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.6964337489127284,
|
|
"grad_norm": 0.5552868606937926,
|
|
"kl": 0.058837890625,
|
|
"learning_rate": 3.9071844115360334e-07,
|
|
"loss": 5.8746740251081064e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4609375,
|
|
"reward_std": 0.20282363891601562,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1201,
|
|
"train_speed(iter/s)": 0.013665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 708.0,
|
|
"completions/mean_length": 232.71875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.697013627138301,
|
|
"grad_norm": 0.4820460750639609,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 3.9053710803789244e-07,
|
|
"loss": 6.792342901462689e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.30859375,
|
|
"reward_std": 0.1640625,
|
|
"rewards/CSTORM/mean": 0.25390625,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 1202,
|
|
"train_speed(iter/s)": 0.013672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 526.0,
|
|
"completions/mean_length": 242.375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.6975935053638735,
|
|
"grad_norm": 0.6819479944400549,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 3.90355672948366e-07,
|
|
"loss": 6.804088479839265e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.703125,
|
|
"reward_std": 0.2907869815826416,
|
|
"rewards/CSTORM/mean": 0.390625,
|
|
"rewards/CSTORM/std": 0.20751149952411652,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 1203,
|
|
"train_speed(iter/s)": 0.013678
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 598.0,
|
|
"completions/mean_length": 249.2890625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.6981733835894463,
|
|
"grad_norm": 0.6948820024907616,
|
|
"kl": 0.0732421875,
|
|
"learning_rate": 3.9017413604516017e-07,
|
|
"loss": 7.32828484615311e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.33390435576438904,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1204,
|
|
"train_speed(iter/s)": 0.013685
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 824.0,
|
|
"completions/mean_length": 242.140625,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.6987532618150188,
|
|
"grad_norm": 0.5961810776089511,
|
|
"kl": 0.068603515625,
|
|
"learning_rate": 3.899924974885005e-07,
|
|
"loss": 6.858028064016253e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.26238328218460083,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1205,
|
|
"train_speed(iter/s)": 0.013692
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 503.0,
|
|
"completions/mean_length": 222.9140625,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.6993331400405914,
|
|
"grad_norm": 0.5366972005558152,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.8981075743870247e-07,
|
|
"loss": 7.289701898116618e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.2777109742164612,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1206,
|
|
"train_speed(iter/s)": 0.013699
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 683.0,
|
|
"completions/mean_length": 256.1484375,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.6999130182661641,
|
|
"grad_norm": 0.4727319575194922,
|
|
"kl": 0.068603515625,
|
|
"learning_rate": 3.8962891605617085e-07,
|
|
"loss": 6.858365668449551e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.61328125,
|
|
"reward_std": 0.2067507952451706,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1207,
|
|
"train_speed(iter/s)": 0.013705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 739.0,
|
|
"completions/mean_length": 246.8515625,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.7004928964917367,
|
|
"grad_norm": 0.5223733322330483,
|
|
"kl": 0.068115234375,
|
|
"learning_rate": 3.8944697350140016e-07,
|
|
"loss": 6.814983498770744e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.2343110889196396,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1208,
|
|
"train_speed(iter/s)": 0.013712
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 992.0,
|
|
"completions/mean_length": 268.375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.7010727747173093,
|
|
"grad_norm": 0.4307373111640315,
|
|
"kl": 0.14404296875,
|
|
"learning_rate": 3.8926492993497404e-07,
|
|
"loss": 0.00014435636694543064,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.2269633561372757,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1209,
|
|
"train_speed(iter/s)": 0.013718
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 665.0,
|
|
"completions/mean_length": 254.3203125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.701652652942882,
|
|
"grad_norm": 0.6325095711046143,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 3.8908278551756554e-07,
|
|
"loss": 7.015799201326445e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.3532869815826416,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1210,
|
|
"train_speed(iter/s)": 0.013723
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 712.0,
|
|
"completions/mean_length": 265.5234375,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.7022325311684546,
|
|
"grad_norm": 0.477884585641886,
|
|
"kl": 0.0703125,
|
|
"learning_rate": 3.8890054040993625e-07,
|
|
"loss": 7.015011215116829e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.63671875,
|
|
"reward_std": 0.1779179871082306,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1211,
|
|
"train_speed(iter/s)": 0.013729
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 830.0,
|
|
"completions/mean_length": 253.0703125,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.7028124093940272,
|
|
"grad_norm": 0.45372805676795075,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 3.8871819477293717e-07,
|
|
"loss": 6.686148844892159e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.171875,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1212,
|
|
"train_speed(iter/s)": 0.013735
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 722.0,
|
|
"completions/mean_length": 259.4921875,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.7033922876195999,
|
|
"grad_norm": 0.553690075725169,
|
|
"kl": 0.0643310546875,
|
|
"learning_rate": 3.8853574876750756e-07,
|
|
"loss": 6.426770414691418e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6640625,
|
|
"reward_std": 0.24809867143630981,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 1213,
|
|
"train_speed(iter/s)": 0.01374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 886.0,
|
|
"completions/mean_length": 276.734375,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.7039721658451725,
|
|
"grad_norm": 0.557130749061568,
|
|
"kl": 0.087646484375,
|
|
"learning_rate": 3.883532025546756e-07,
|
|
"loss": 8.753377915127203e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.2975226938724518,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1214,
|
|
"train_speed(iter/s)": 0.013746
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 548.0,
|
|
"completions/mean_length": 231.59375,
|
|
"completions/min_length": 67.0,
|
|
"epoch": 0.7045520440707451,
|
|
"grad_norm": 0.5287082237279533,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.8817055629555766e-07,
|
|
"loss": 7.29908497305587e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.7734375,
|
|
"reward_std": 0.1791265904903412,
|
|
"rewards/CSTORM/mean": 0.4140625,
|
|
"rewards/CSTORM/std": 0.1893770843744278,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.859375,
|
|
"rewards/VQAORM/std": 0.3490002751350403,
|
|
"step": 1215,
|
|
"train_speed(iter/s)": 0.013753
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 661.0,
|
|
"completions/mean_length": 245.3984375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.7051319222963178,
|
|
"grad_norm": 0.5990603821108559,
|
|
"kl": 0.07177734375,
|
|
"learning_rate": 3.8798781015135867e-07,
|
|
"loss": 7.182401895988733e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3671875,
|
|
"reward_std": 0.38682234287261963,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1216,
|
|
"train_speed(iter/s)": 0.01376
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 525.0,
|
|
"completions/mean_length": 215.65625,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.7057118005218904,
|
|
"grad_norm": 0.6608957691524923,
|
|
"kl": 0.073974609375,
|
|
"learning_rate": 3.878049642833715e-07,
|
|
"loss": 7.418380118906498e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.61328125,
|
|
"reward_std": 0.3550376296043396,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1217,
|
|
"train_speed(iter/s)": 0.013767
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 710.0,
|
|
"completions/mean_length": 238.921875,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.706291678747463,
|
|
"grad_norm": 0.5192353597268101,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 3.8762201885297725e-07,
|
|
"loss": 7.034502050373703e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.21102428436279297,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1218,
|
|
"train_speed(iter/s)": 0.013773
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 540.0,
|
|
"completions/mean_length": 244.2890625,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.7068715569730356,
|
|
"grad_norm": 0.5118825215573666,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 3.8743897402164457e-07,
|
|
"loss": 7.055913738440722e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.1989382952451706,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1219,
|
|
"train_speed(iter/s)": 0.01378
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 750.0,
|
|
"completions/mean_length": 246.171875,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.7074514351986083,
|
|
"grad_norm": 0.4469901219704044,
|
|
"kl": 0.071533203125,
|
|
"learning_rate": 3.8725582995093017e-07,
|
|
"loss": 7.15376518201083e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.18481916189193726,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1220,
|
|
"train_speed(iter/s)": 0.013785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 659.0,
|
|
"completions/mean_length": 227.5078125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.7080313134241809,
|
|
"grad_norm": 0.4335747964971668,
|
|
"kl": 0.070068359375,
|
|
"learning_rate": 3.8707258680247806e-07,
|
|
"loss": 7.030351844150573e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.15853539109230042,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1221,
|
|
"train_speed(iter/s)": 0.013792
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 670.0,
|
|
"completions/mean_length": 255.328125,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.7086111916497535,
|
|
"grad_norm": 0.5708142019179019,
|
|
"kl": 0.061767578125,
|
|
"learning_rate": 3.8688924473802013e-07,
|
|
"loss": 6.180617492645979e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.2536258101463318,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1222,
|
|
"train_speed(iter/s)": 0.013799
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 646.0,
|
|
"completions/mean_length": 234.3203125,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.7091910698753262,
|
|
"grad_norm": 7.211415986608468,
|
|
"kl": 0.237060546875,
|
|
"learning_rate": 3.867058039193751e-07,
|
|
"loss": 0.00023665520711801946,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37890625,
|
|
"reward_std": 0.13476593792438507,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 1223,
|
|
"train_speed(iter/s)": 0.01374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 875.0,
|
|
"completions/mean_length": 283.578125,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.7097709481008988,
|
|
"grad_norm": 0.5717276419535121,
|
|
"kl": 0.05859375,
|
|
"learning_rate": 3.865222645084491e-07,
|
|
"loss": 5.8648041886044666e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.3035656809806824,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1224,
|
|
"train_speed(iter/s)": 0.013746
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 692.0,
|
|
"completions/mean_length": 226.1640625,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.7103508263264714,
|
|
"grad_norm": 0.39228938274907393,
|
|
"kl": 0.0673828125,
|
|
"learning_rate": 3.863386266672352e-07,
|
|
"loss": 6.728592416038737e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.12202189117670059,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1225,
|
|
"train_speed(iter/s)": 0.013753
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 975.0,
|
|
"completions/mean_length": 264.453125,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.7109307045520441,
|
|
"grad_norm": 0.49411290922361284,
|
|
"kl": 0.0618896484375,
|
|
"learning_rate": 3.861548905578134e-07,
|
|
"loss": 6.19473066763021e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.20122367143630981,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1226,
|
|
"train_speed(iter/s)": 0.013758
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1511.0,
|
|
"completions/mean_length": 274.9453125,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.7115105827776167,
|
|
"grad_norm": 0.48668856235024455,
|
|
"kl": 0.06396484375,
|
|
"learning_rate": 3.8597105634235036e-07,
|
|
"loss": 6.40318903606385e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.17252269387245178,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1227,
|
|
"train_speed(iter/s)": 0.013762
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1145.0,
|
|
"completions/mean_length": 274.59375,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.7120904610031893,
|
|
"grad_norm": 0.5242539837760863,
|
|
"kl": 0.07958984375,
|
|
"learning_rate": 3.857871241830994e-07,
|
|
"loss": 7.983671093825251e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.54296875,
|
|
"reward_std": 0.22452935576438904,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1228,
|
|
"train_speed(iter/s)": 0.013767
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1021.0,
|
|
"completions/mean_length": 233.8828125,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.712670339228762,
|
|
"grad_norm": 0.4542980357119393,
|
|
"kl": 0.077392578125,
|
|
"learning_rate": 3.856030942424004e-07,
|
|
"loss": 7.725274190306664e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.1989382952451706,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1229,
|
|
"train_speed(iter/s)": 0.013772
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 682.0,
|
|
"completions/mean_length": 243.7265625,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.7132502174543346,
|
|
"grad_norm": 0.43995789769836197,
|
|
"kl": 0.07080078125,
|
|
"learning_rate": 3.854189666826792e-07,
|
|
"loss": 7.084225217113271e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.14709708094596863,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1230,
|
|
"train_speed(iter/s)": 0.013779
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 914.0,
|
|
"completions/mean_length": 270.5625,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.7138300956799072,
|
|
"grad_norm": 0.6047146560008921,
|
|
"kl": 0.0626220703125,
|
|
"learning_rate": 3.85234741666448e-07,
|
|
"loss": 6.259871588554233e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.25442007184028625,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1231,
|
|
"train_speed(iter/s)": 0.013772
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 685.0,
|
|
"completions/mean_length": 254.59375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.7144099739054799,
|
|
"grad_norm": 0.6086744362953863,
|
|
"kl": 0.0615234375,
|
|
"learning_rate": 3.8505041935630516e-07,
|
|
"loss": 6.154800939839333e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.30477428436279297,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1232,
|
|
"train_speed(iter/s)": 0.013779
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 851.0,
|
|
"completions/mean_length": 247.6015625,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.7149898521310525,
|
|
"grad_norm": 0.4122784998516288,
|
|
"kl": 0.069580078125,
|
|
"learning_rate": 3.848659999149347e-07,
|
|
"loss": 6.962371116969734e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.1713140904903412,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1233,
|
|
"train_speed(iter/s)": 0.013785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 784.0,
|
|
"completions/mean_length": 218.3828125,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.7155697303566251,
|
|
"grad_norm": 1.2660758441487192,
|
|
"kl": 9.724365234375,
|
|
"learning_rate": 3.846814835051064e-07,
|
|
"loss": 0.009717755019664764,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.28803783655166626,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1234,
|
|
"train_speed(iter/s)": 0.013727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 820.0,
|
|
"completions/mean_length": 262.234375,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.7161496085821978,
|
|
"grad_norm": 0.48844943447346184,
|
|
"kl": 0.0634765625,
|
|
"learning_rate": 3.8449687028967575e-07,
|
|
"loss": 6.346340524032712e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.42578125,
|
|
"reward_std": 0.2542734742164612,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1235,
|
|
"train_speed(iter/s)": 0.013721
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 843.0,
|
|
"completions/mean_length": 227.390625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.7167294868077704,
|
|
"grad_norm": 0.5585605898939288,
|
|
"kl": 0.071533203125,
|
|
"learning_rate": 3.843121604315835e-07,
|
|
"loss": 7.155411003623158e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.2348908632993698,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1236,
|
|
"train_speed(iter/s)": 0.013728
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 758.0,
|
|
"completions/mean_length": 272.7578125,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.717309365033343,
|
|
"grad_norm": 0.426288582924761,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 3.841273540938559e-07,
|
|
"loss": 6.786126323277131e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55078125,
|
|
"reward_std": 0.16216117143630981,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1237,
|
|
"train_speed(iter/s)": 0.013734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1162.0,
|
|
"completions/mean_length": 250.3984375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.7178892432589157,
|
|
"grad_norm": 0.5424067233092368,
|
|
"kl": 0.067138671875,
|
|
"learning_rate": 3.8394245143960415e-07,
|
|
"loss": 6.717237556586042e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.203640878200531,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1238,
|
|
"train_speed(iter/s)": 0.013738
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 824.0,
|
|
"completions/mean_length": 268.8515625,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.7184691214844883,
|
|
"grad_norm": 0.5150421267576568,
|
|
"kl": 0.0538330078125,
|
|
"learning_rate": 3.837574526320246e-07,
|
|
"loss": 5.383483221521601e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.2475377768278122,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 1239,
|
|
"train_speed(iter/s)": 0.013744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 839.0,
|
|
"completions/mean_length": 292.390625,
|
|
"completions/min_length": 103.0,
|
|
"epoch": 0.7190489997100609,
|
|
"grad_norm": 0.5253439110221901,
|
|
"kl": 0.0628662109375,
|
|
"learning_rate": 3.835723578343984e-07,
|
|
"loss": 6.288645090535283e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62890625,
|
|
"reward_std": 0.1713140904903412,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1240,
|
|
"train_speed(iter/s)": 0.01374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1008.0,
|
|
"completions/mean_length": 290.8671875,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.7196288779356336,
|
|
"grad_norm": 0.4221643850468248,
|
|
"kl": 0.0587158203125,
|
|
"learning_rate": 3.833871672100914e-07,
|
|
"loss": 5.8622448705136776e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.1218712106347084,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 1241,
|
|
"train_speed(iter/s)": 0.013744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 774.0,
|
|
"completions/mean_length": 258.4140625,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.7202087561612062,
|
|
"grad_norm": 0.694164642833187,
|
|
"kl": 0.0650634765625,
|
|
"learning_rate": 3.832018809225542e-07,
|
|
"loss": 6.515422865049914e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3359375,
|
|
"reward_std": 0.42140087485313416,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 1242,
|
|
"train_speed(iter/s)": 0.013751
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 796.0,
|
|
"completions/mean_length": 263.6484375,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.7207886343867788,
|
|
"grad_norm": 0.43926545094021974,
|
|
"kl": 0.06201171875,
|
|
"learning_rate": 3.8301649913532156e-07,
|
|
"loss": 6.206044054124504e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.2169804871082306,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1243,
|
|
"train_speed(iter/s)": 0.013755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2013.0,
|
|
"completions/mean_length": 289.1796875,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.7213685126123514,
|
|
"grad_norm": 0.4855463690106904,
|
|
"kl": 0.0673828125,
|
|
"learning_rate": 3.828310220120128e-07,
|
|
"loss": 6.73989561619237e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.2301882952451706,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1244,
|
|
"train_speed(iter/s)": 0.013756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 543.0,
|
|
"completions/mean_length": 249.0625,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.7219483908379241,
|
|
"grad_norm": 0.5045490102914383,
|
|
"kl": 0.072021484375,
|
|
"learning_rate": 3.826454497163311e-07,
|
|
"loss": 7.214256038423628e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.22988693416118622,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1245,
|
|
"train_speed(iter/s)": 0.013763
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/mean_length": 270.8203125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.7225282690634967,
|
|
"grad_norm": 0.4727218623871027,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 3.8245978241206395e-07,
|
|
"loss": 6.765241414541379e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.640625,
|
|
"reward_std": 0.1951618194580078,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 1246,
|
|
"train_speed(iter/s)": 0.013769
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 650.0,
|
|
"completions/mean_length": 261.4296875,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.7231081472890692,
|
|
"grad_norm": 0.5680591885704193,
|
|
"kl": 0.068603515625,
|
|
"learning_rate": 3.822740202630824e-07,
|
|
"loss": 6.849081546533853e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.28510916233062744,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1247,
|
|
"train_speed(iter/s)": 0.013775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 921.0,
|
|
"completions/mean_length": 259.3515625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.723688025514642,
|
|
"grad_norm": 0.43151617821158617,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 3.8208816343334154e-07,
|
|
"loss": 6.569655670318753e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.15853539109230042,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1248,
|
|
"train_speed(iter/s)": 0.013781
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 613.0,
|
|
"completions/mean_length": 239.90625,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.7242679037402145,
|
|
"grad_norm": 0.5054220019603719,
|
|
"kl": 0.07421875,
|
|
"learning_rate": 3.819022120868796e-07,
|
|
"loss": 7.416173320962116e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.12564769387245178,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1249,
|
|
"train_speed(iter/s)": 0.013788
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 622.0,
|
|
"completions/mean_length": 233.3125,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.7248477819657871,
|
|
"grad_norm": 0.5124732829485505,
|
|
"kl": 0.069580078125,
|
|
"learning_rate": 3.8171616638781866e-07,
|
|
"loss": 6.946882058400661e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.71875,
|
|
"reward_std": 0.2348908632993698,
|
|
"rewards/CSTORM/mean": 0.375,
|
|
"rewards/CSTORM/std": 0.2173570692539215,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.84375,
|
|
"rewards/VQAORM/std": 0.3645188808441162,
|
|
"step": 1250,
|
|
"train_speed(iter/s)": 0.013794
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1008.0,
|
|
"completions/mean_length": 277.921875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.7254276601913598,
|
|
"grad_norm": 0.5161251588544954,
|
|
"kl": 0.065673828125,
|
|
"learning_rate": 3.8153002650036385e-07,
|
|
"loss": 6.566548836417496e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.390625,
|
|
"reward_std": 0.140625,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1251,
|
|
"train_speed(iter/s)": 0.0138
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1726.0,
|
|
"completions/mean_length": 260.109375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.7260075384169324,
|
|
"grad_norm": 0.6371696154182197,
|
|
"kl": 0.074951171875,
|
|
"learning_rate": 3.8134379258880335e-07,
|
|
"loss": 7.504236418753862e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.28858837485313416,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1252,
|
|
"train_speed(iter/s)": 0.013802
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 816.0,
|
|
"completions/mean_length": 256.1796875,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.726587416642505,
|
|
"grad_norm": 0.491669367156387,
|
|
"kl": 0.0631103515625,
|
|
"learning_rate": 3.8115746481750863e-07,
|
|
"loss": 6.316885992418975e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.2181890904903412,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1253,
|
|
"train_speed(iter/s)": 0.013808
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 510.0,
|
|
"completions/mean_length": 234.109375,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.7271672948680777,
|
|
"grad_norm": 0.422039734806553,
|
|
"kl": 0.068359375,
|
|
"learning_rate": 3.809710433509335e-07,
|
|
"loss": 6.830796337453648e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.15732678771018982,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1254,
|
|
"train_speed(iter/s)": 0.013815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 609.0,
|
|
"completions/mean_length": 236.4921875,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.7277471730936503,
|
|
"grad_norm": 0.5387886184902522,
|
|
"kl": 0.076416015625,
|
|
"learning_rate": 3.8078452835361515e-07,
|
|
"loss": 7.653395005036145e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.2566906809806824,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1255,
|
|
"train_speed(iter/s)": 0.013821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 884.0,
|
|
"completions/mean_length": 299.5859375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.7283270513192229,
|
|
"grad_norm": 0.376296222047008,
|
|
"kl": 0.0621337890625,
|
|
"learning_rate": 3.8059791999017265e-07,
|
|
"loss": 6.213872256921604e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.19397208094596863,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1256,
|
|
"train_speed(iter/s)": 0.013827
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 913.0,
|
|
"completions/mean_length": 242.5390625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.7289069295447956,
|
|
"grad_norm": 0.5555216874748499,
|
|
"kl": 0.07373046875,
|
|
"learning_rate": 3.8041121842530803e-07,
|
|
"loss": 7.368941442109644e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.52734375,
|
|
"reward_std": 0.21982678771018982,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1257,
|
|
"train_speed(iter/s)": 0.013833
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 930.0,
|
|
"completions/mean_length": 250.453125,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.7294868077703682,
|
|
"grad_norm": 0.3873284158984671,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 3.8022442382380514e-07,
|
|
"loss": 7.047138933558017e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3203125,
|
|
"reward_std": 0.155538409948349,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 1258,
|
|
"train_speed(iter/s)": 0.013839
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 734.0,
|
|
"completions/mean_length": 238.671875,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.7300666859959408,
|
|
"grad_norm": 0.501455875366956,
|
|
"kl": 0.075439453125,
|
|
"learning_rate": 3.800375363505302e-07,
|
|
"loss": 7.531889423262328e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.17657756805419922,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1259,
|
|
"train_speed(iter/s)": 0.013845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 886.0,
|
|
"completions/mean_length": 263.2265625,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.7306465642215135,
|
|
"grad_norm": 0.5237263953659416,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 3.7985055617043135e-07,
|
|
"loss": 6.590576231246814e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53125,
|
|
"reward_std": 0.2751619815826416,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1260,
|
|
"train_speed(iter/s)": 0.013851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 887.0,
|
|
"completions/mean_length": 281.0859375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.7312264424470861,
|
|
"grad_norm": 0.45259941708700124,
|
|
"kl": 0.0596923828125,
|
|
"learning_rate": 3.796634834485384e-07,
|
|
"loss": 5.966196476947516e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.23139688372612,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1261,
|
|
"train_speed(iter/s)": 0.013857
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 625.0,
|
|
"completions/mean_length": 228.59375,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.7318063206726587,
|
|
"grad_norm": 0.405768268302829,
|
|
"kl": 0.07275390625,
|
|
"learning_rate": 3.794763183499632e-07,
|
|
"loss": 7.294467650353909e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.16997367143630981,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1262,
|
|
"train_speed(iter/s)": 0.013863
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 933.0,
|
|
"completions/mean_length": 253.8359375,
|
|
"completions/min_length": 51.0,
|
|
"epoch": 0.7323861988982314,
|
|
"grad_norm": 0.5200364275228072,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 3.792890610398987e-07,
|
|
"loss": 6.994300201768056e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.2270595133304596,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1263,
|
|
"train_speed(iter/s)": 0.013855
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 772.0,
|
|
"completions/mean_length": 275.5859375,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.732966077123804,
|
|
"grad_norm": 0.5299108944418246,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 3.791017116836196e-07,
|
|
"loss": 6.437924457713962e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.2133883535861969,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1264,
|
|
"train_speed(iter/s)": 0.013861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 579.0,
|
|
"completions/mean_length": 248.5390625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.7335459553493766,
|
|
"grad_norm": 0.5033924502997281,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 3.789142704464815e-07,
|
|
"loss": 6.925337947905064e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.2145632952451706,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1265,
|
|
"train_speed(iter/s)": 0.013868
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 682.0,
|
|
"completions/mean_length": 239.1171875,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.7341258335749493,
|
|
"grad_norm": 0.4139766606707545,
|
|
"kl": 0.071533203125,
|
|
"learning_rate": 3.7872673749392135e-07,
|
|
"loss": 7.166211435105652e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.14466102421283722,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 1266,
|
|
"train_speed(iter/s)": 0.013875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 514.0,
|
|
"completions/mean_length": 215.3203125,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.7347057118005219,
|
|
"grad_norm": 9.502490655082438,
|
|
"kl": 0.3662109375,
|
|
"learning_rate": 3.785391129914571e-07,
|
|
"loss": 0.0003671108279377222,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6875,
|
|
"reward_std": 0.3270031809806824,
|
|
"rewards/CSTORM/mean": 0.38671875,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.8046875,
|
|
"rewards/VQAORM/std": 0.3979988098144531,
|
|
"step": 1267,
|
|
"train_speed(iter/s)": 0.013816
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 914.0,
|
|
"completions/mean_length": 240.3515625,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.7352855900260945,
|
|
"grad_norm": 0.5631022805064522,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 3.783513971046872e-07,
|
|
"loss": 7.502762309741229e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.215621218085289,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1268,
|
|
"train_speed(iter/s)": 0.013821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 555.0,
|
|
"completions/mean_length": 234.71875,
|
|
"completions/min_length": 104.0,
|
|
"epoch": 0.7358654682516671,
|
|
"grad_norm": 0.504666745202181,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 3.78163589999291e-07,
|
|
"loss": 6.875280087115243e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.21119704842567444,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1269,
|
|
"train_speed(iter/s)": 0.013827
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 709.0,
|
|
"completions/mean_length": 235.828125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.7364453464772398,
|
|
"grad_norm": 0.4880932201737885,
|
|
"kl": 0.073486328125,
|
|
"learning_rate": 3.779756918410283e-07,
|
|
"loss": 7.348546932917088e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6640625,
|
|
"reward_std": 0.20377269387245178,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 1270,
|
|
"train_speed(iter/s)": 0.013834
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 897.0,
|
|
"completions/mean_length": 228.40625,
|
|
"completions/min_length": 67.0,
|
|
"epoch": 0.7370252247028124,
|
|
"grad_norm": 0.534733557780898,
|
|
"kl": 0.10546875,
|
|
"learning_rate": 3.7778770279573936e-07,
|
|
"loss": 0.00010556649067439139,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.16755647957324982,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1271,
|
|
"train_speed(iter/s)": 0.013839
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 690.0,
|
|
"completions/mean_length": 232.046875,
|
|
"completions/min_length": 61.0,
|
|
"epoch": 0.737605102928385,
|
|
"grad_norm": 0.4796486699055547,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 3.7759962302934435e-07,
|
|
"loss": 6.936825229786336e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.19612565636634827,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1272,
|
|
"train_speed(iter/s)": 0.013846
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 996.0,
|
|
"completions/mean_length": 257.1328125,
|
|
"completions/min_length": 67.0,
|
|
"epoch": 0.7381849811539577,
|
|
"grad_norm": 0.36427693821082047,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.7741145270784385e-07,
|
|
"loss": 7.294840179383755e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.1130007952451706,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1273,
|
|
"train_speed(iter/s)": 0.01384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1092.0,
|
|
"completions/mean_length": 269.6484375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.7387648593795303,
|
|
"grad_norm": 0.47315987437271173,
|
|
"kl": 0.07275390625,
|
|
"learning_rate": 3.7722319199731816e-07,
|
|
"loss": 7.278057455550879e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.24525237083435059,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1274,
|
|
"train_speed(iter/s)": 0.013845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 831.0,
|
|
"completions/mean_length": 253.71875,
|
|
"completions/min_length": 115.0,
|
|
"epoch": 0.7393447376051029,
|
|
"grad_norm": 0.5981858591780728,
|
|
"kl": 0.0654296875,
|
|
"learning_rate": 3.7703484106392745e-07,
|
|
"loss": 6.543520430568606e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.299141526222229,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1275,
|
|
"train_speed(iter/s)": 0.013851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 533.0,
|
|
"completions/mean_length": 243.8515625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.7399246158306756,
|
|
"grad_norm": 0.4538208424448428,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 3.7684640007391157e-07,
|
|
"loss": 6.931401003384963e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.16794784367084503,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1276,
|
|
"train_speed(iter/s)": 0.013858
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2279.0,
|
|
"completions/mean_length": 253.3515625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.7405044940562482,
|
|
"grad_norm": 0.4669597603371541,
|
|
"kl": 0.072265625,
|
|
"learning_rate": 3.766578691935897e-07,
|
|
"loss": 7.222677231766284e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.18197289109230042,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1277,
|
|
"train_speed(iter/s)": 0.013851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 773.0,
|
|
"completions/mean_length": 275.2734375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.7410843722818208,
|
|
"grad_norm": 0.5870745077504453,
|
|
"kl": 0.068359375,
|
|
"learning_rate": 3.764692485893604e-07,
|
|
"loss": 6.813941581640393e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.2805572748184204,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 1278,
|
|
"train_speed(iter/s)": 0.013856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 835.0,
|
|
"completions/mean_length": 225.59375,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.7416642505073935,
|
|
"grad_norm": 0.6141412975582788,
|
|
"kl": 0.075927734375,
|
|
"learning_rate": 3.7628053842770153e-07,
|
|
"loss": 7.579761586384848e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.24766957759857178,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1279,
|
|
"train_speed(iter/s)": 0.013862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1441.0,
|
|
"completions/mean_length": 287.90625,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.7422441287329661,
|
|
"grad_norm": 0.5614754523836275,
|
|
"kl": 0.0670166015625,
|
|
"learning_rate": 3.7609173887517e-07,
|
|
"loss": 6.698493234580383e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.234375,
|
|
"reward_std": 0.2679103910923004,
|
|
"rewards/CSTORM/mean": 0.2265625,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5078125,
|
|
"rewards/VQAORM/std": 0.5019033551216125,
|
|
"step": 1280,
|
|
"train_speed(iter/s)": 0.013866
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 939.0,
|
|
"completions/mean_length": 275.78125,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.7428240069585387,
|
|
"grad_norm": 0.5170940506669992,
|
|
"kl": 0.0703125,
|
|
"learning_rate": 3.759028500984014e-07,
|
|
"loss": 7.032359280856326e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.1989382952451706,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1281,
|
|
"train_speed(iter/s)": 0.013872
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 638.0,
|
|
"completions/mean_length": 228.4609375,
|
|
"completions/min_length": 56.0,
|
|
"epoch": 0.7434038851841114,
|
|
"grad_norm": 0.5519084731669337,
|
|
"kl": 0.074951171875,
|
|
"learning_rate": 3.7571387226411026e-07,
|
|
"loss": 7.506242400268093e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.2697666883468628,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1282,
|
|
"train_speed(iter/s)": 0.013877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 922.0,
|
|
"completions/mean_length": 270.7578125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.743983763409684,
|
|
"grad_norm": 0.5797085040719128,
|
|
"kl": 0.0628662109375,
|
|
"learning_rate": 3.7552480553908976e-07,
|
|
"loss": 6.290932651609182e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.3513856530189514,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1283,
|
|
"train_speed(iter/s)": 0.013882
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1130.0,
|
|
"completions/mean_length": 280.875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.7445636416352566,
|
|
"grad_norm": 0.4718359290211751,
|
|
"kl": 0.06591796875,
|
|
"learning_rate": 3.753356500902114e-07,
|
|
"loss": 6.609478441532701e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.18723636865615845,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1284,
|
|
"train_speed(iter/s)": 0.013887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1200.0,
|
|
"completions/mean_length": 254.84375,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.7451435198608293,
|
|
"grad_norm": 0.5349330364576658,
|
|
"kl": 0.0712890625,
|
|
"learning_rate": 3.751464060844249e-07,
|
|
"loss": 7.11806133040227e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.25038406252861023,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1285,
|
|
"train_speed(iter/s)": 0.013878
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1133.0,
|
|
"completions/mean_length": 233.4296875,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.7457233980864019,
|
|
"grad_norm": 0.5024339853922203,
|
|
"kl": 0.068359375,
|
|
"learning_rate": 3.749570736887584e-07,
|
|
"loss": 6.840075366199017e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.1833132803440094,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1286,
|
|
"train_speed(iter/s)": 0.013882
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 808.0,
|
|
"completions/mean_length": 259.3984375,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.7463032763119745,
|
|
"grad_norm": 0.5670666733204288,
|
|
"kl": 0.08349609375,
|
|
"learning_rate": 3.747676530703179e-07,
|
|
"loss": 8.371843432541937e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.32281649112701416,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1287,
|
|
"train_speed(iter/s)": 0.013889
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 740.0,
|
|
"completions/mean_length": 246.6328125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.7468831545375472,
|
|
"grad_norm": 0.5496719296704075,
|
|
"kl": 0.066650390625,
|
|
"learning_rate": 3.745781443962872e-07,
|
|
"loss": 6.684838444925845e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66015625,
|
|
"reward_std": 0.325662761926651,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 1288,
|
|
"train_speed(iter/s)": 0.013895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 550.0,
|
|
"completions/mean_length": 224.25,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.7474630327631198,
|
|
"grad_norm": 0.5711360887288683,
|
|
"kl": 0.0732421875,
|
|
"learning_rate": 3.7438854783392805e-07,
|
|
"loss": 7.31556792743504e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.2607266902923584,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1289,
|
|
"train_speed(iter/s)": 0.013901
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 636.0,
|
|
"completions/mean_length": 248.2734375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.7480429109886924,
|
|
"grad_norm": 0.43771897842520135,
|
|
"kl": 0.069091796875,
|
|
"learning_rate": 3.7419886355057963e-07,
|
|
"loss": 6.89680891809985e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.20299318432807922,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1290,
|
|
"train_speed(iter/s)": 0.013908
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1336.0,
|
|
"completions/mean_length": 252.5234375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.748622789214265,
|
|
"grad_norm": 0.41536240591877854,
|
|
"kl": 0.064697265625,
|
|
"learning_rate": 3.7400909171365843e-07,
|
|
"loss": 6.475206464529037e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.20618988573551178,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1291,
|
|
"train_speed(iter/s)": 0.013912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 546.0,
|
|
"completions/mean_length": 229.9453125,
|
|
"completions/min_length": 113.0,
|
|
"epoch": 0.7492026674398377,
|
|
"grad_norm": 0.38110068675242553,
|
|
"kl": 0.07373046875,
|
|
"learning_rate": 3.7381923249065835e-07,
|
|
"loss": 7.369231025222689e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6484375,
|
|
"reward_std": 0.14368988573551178,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 1292,
|
|
"train_speed(iter/s)": 0.013918
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 644.0,
|
|
"completions/mean_length": 238.640625,
|
|
"completions/min_length": 61.0,
|
|
"epoch": 0.7497825456654102,
|
|
"grad_norm": 0.5423515487434889,
|
|
"kl": 0.0703125,
|
|
"learning_rate": 3.736292860491504e-07,
|
|
"loss": 7.036793977022171e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.224661186337471,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1293,
|
|
"train_speed(iter/s)": 0.013925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 592.0,
|
|
"completions/mean_length": 236.3125,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.7503624238909828,
|
|
"grad_norm": 0.45048939527977544,
|
|
"kl": 0.075439453125,
|
|
"learning_rate": 3.734392525567826e-07,
|
|
"loss": 7.546372944489121e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.1713140904903412,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1294,
|
|
"train_speed(iter/s)": 0.013931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 552.0,
|
|
"completions/mean_length": 231.59375,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.7509423021165555,
|
|
"grad_norm": 0.5388136334519962,
|
|
"kl": 0.073974609375,
|
|
"learning_rate": 3.732491321812798e-07,
|
|
"loss": 7.392032421194017e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.2078087031841278,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1295,
|
|
"train_speed(iter/s)": 0.013938
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 564.0,
|
|
"completions/mean_length": 221.9609375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.7515221803421281,
|
|
"grad_norm": 0.4857706712655361,
|
|
"kl": 0.07763671875,
|
|
"learning_rate": 3.730589250904435e-07,
|
|
"loss": 7.746646588202566e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6484375,
|
|
"reward_std": 0.20541039109230042,
|
|
"rewards/CSTORM/mean": 0.375,
|
|
"rewards/CSTORM/std": 0.2173570692539215,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1296,
|
|
"train_speed(iter/s)": 0.013944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 671.0,
|
|
"completions/mean_length": 248.453125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.7521020585677007,
|
|
"grad_norm": 0.4910141557530478,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.728686314521516e-07,
|
|
"loss": 7.27848382666707e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.23288390040397644,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1297,
|
|
"train_speed(iter/s)": 0.01395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 659.0,
|
|
"completions/mean_length": 261.3203125,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.7526819367932734,
|
|
"grad_norm": 0.516939615812258,
|
|
"kl": 0.07275390625,
|
|
"learning_rate": 3.7267825143435864e-07,
|
|
"loss": 7.279634883161634e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.1959601789712906,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1298,
|
|
"train_speed(iter/s)": 0.013955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 611.0,
|
|
"completions/mean_length": 234.421875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.753261815018846,
|
|
"grad_norm": 0.5058080398336317,
|
|
"kl": 0.075439453125,
|
|
"learning_rate": 3.724877852050952e-07,
|
|
"loss": 7.55532382754609e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.19504886865615845,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1299,
|
|
"train_speed(iter/s)": 0.013962
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 600.0,
|
|
"completions/mean_length": 242.8203125,
|
|
"completions/min_length": 105.0,
|
|
"epoch": 0.7538416932444186,
|
|
"grad_norm": 0.5801235539224963,
|
|
"kl": 0.072509765625,
|
|
"learning_rate": 3.7229723293246817e-07,
|
|
"loss": 7.250114867929369e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.23828125,
|
|
"reward_std": 0.2404179871082306,
|
|
"rewards/CSTORM/mean": 0.23828125,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 1300,
|
|
"train_speed(iter/s)": 0.013967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 797.0,
|
|
"completions/mean_length": 257.8203125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.7544215714699913,
|
|
"grad_norm": 0.5872004671570086,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 3.7210659478466003e-07,
|
|
"loss": 6.789692997699603e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.2883697748184204,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1301,
|
|
"train_speed(iter/s)": 0.013968
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 533.0,
|
|
"completions/mean_length": 226.828125,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.7550014496955639,
|
|
"grad_norm": 0.6086235623561798,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 3.719158709299296e-07,
|
|
"loss": 6.977548764552921e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.2829744815826416,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1302,
|
|
"train_speed(iter/s)": 0.013974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 755.0,
|
|
"completions/mean_length": 240.984375,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.7555813279211365,
|
|
"grad_norm": 0.5790708997692067,
|
|
"kl": 0.07763671875,
|
|
"learning_rate": 3.7172506153661076e-07,
|
|
"loss": 7.760948210489005e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33984375,
|
|
"reward_std": 0.258158802986145,
|
|
"rewards/CSTORM/mean": 0.24609375,
|
|
"rewards/CSTORM/std": 0.2509516775608063,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1303,
|
|
"train_speed(iter/s)": 0.01398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 636.0,
|
|
"completions/mean_length": 230.375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.7561612061467092,
|
|
"grad_norm": 0.5954828586278378,
|
|
"kl": 0.08837890625,
|
|
"learning_rate": 3.715341667731132e-07,
|
|
"loss": 8.85108020156622e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.23666039109230042,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1304,
|
|
"train_speed(iter/s)": 0.013985
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 480.0,
|
|
"completions/mean_length": 231.796875,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.7567410843722818,
|
|
"grad_norm": 0.49774981537120805,
|
|
"kl": 0.07568359375,
|
|
"learning_rate": 3.7134318680792184e-07,
|
|
"loss": 7.583748083561659e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.2494390904903412,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1305,
|
|
"train_speed(iter/s)": 0.013992
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 883.0,
|
|
"completions/mean_length": 236.7421875,
|
|
"completions/min_length": 107.0,
|
|
"epoch": 0.7573209625978544,
|
|
"grad_norm": 0.43542645890874027,
|
|
"kl": 0.076171875,
|
|
"learning_rate": 3.7115212180959695e-07,
|
|
"loss": 7.61647243052721e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.1791265904903412,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 1306,
|
|
"train_speed(iter/s)": 0.013998
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 790.0,
|
|
"completions/mean_length": 218.984375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.7579008408234271,
|
|
"grad_norm": 0.45208383875901015,
|
|
"kl": 0.077880859375,
|
|
"learning_rate": 3.7096097194677373e-07,
|
|
"loss": 7.808564987499267e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.14696526527404785,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1307,
|
|
"train_speed(iter/s)": 0.014004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 601.0,
|
|
"completions/mean_length": 240.3125,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.7584807190489997,
|
|
"grad_norm": 0.5684489047267923,
|
|
"kl": 0.06494140625,
|
|
"learning_rate": 3.707697373881623e-07,
|
|
"loss": 6.507506623165682e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.24675826728343964,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1308,
|
|
"train_speed(iter/s)": 0.014009
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 543.0,
|
|
"completions/mean_length": 225.078125,
|
|
"completions/min_length": 71.0,
|
|
"epoch": 0.7590605972745723,
|
|
"grad_norm": 0.5471707498300638,
|
|
"kl": 0.07421875,
|
|
"learning_rate": 3.705784183025475e-07,
|
|
"loss": 7.42782503948547e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.24106568098068237,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1309,
|
|
"train_speed(iter/s)": 0.014016
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 872.0,
|
|
"completions/mean_length": 244.6875,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.759640475500145,
|
|
"grad_norm": 0.42566484620323264,
|
|
"kl": 0.07275390625,
|
|
"learning_rate": 3.7038701485878896e-07,
|
|
"loss": 7.271418144227937e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.11207061260938644,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1310,
|
|
"train_speed(iter/s)": 0.014022
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1027.0,
|
|
"completions/mean_length": 272.71875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.7602203537257176,
|
|
"grad_norm": 0.5211112890232085,
|
|
"kl": 0.073486328125,
|
|
"learning_rate": 3.7019552722582067e-07,
|
|
"loss": 7.341097079915926e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2301882952451706,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1311,
|
|
"train_speed(iter/s)": 0.014027
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 605.0,
|
|
"completions/mean_length": 240.484375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.7608002319512902,
|
|
"grad_norm": 0.44413900120222066,
|
|
"kl": 0.060546875,
|
|
"learning_rate": 3.700039555726505e-07,
|
|
"loss": 6.0600363212870434e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.39453125,
|
|
"reward_std": 0.18033519387245178,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6015625,
|
|
"rewards/VQAORM/std": 0.4915000796318054,
|
|
"step": 1312,
|
|
"train_speed(iter/s)": 0.014022
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 799.0,
|
|
"completions/mean_length": 222.265625,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.7613801101768629,
|
|
"grad_norm": 0.5348333111719684,
|
|
"kl": 0.07080078125,
|
|
"learning_rate": 3.698123000683613e-07,
|
|
"loss": 7.06451028236188e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.1911257952451706,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1313,
|
|
"train_speed(iter/s)": 0.014029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 674.0,
|
|
"completions/mean_length": 232.296875,
|
|
"completions/min_length": 61.0,
|
|
"epoch": 0.7619599884024355,
|
|
"grad_norm": 0.28994618784920134,
|
|
"kl": 0.072509765625,
|
|
"learning_rate": 3.6962056088210913e-07,
|
|
"loss": 7.253669900819659e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.68359375,
|
|
"reward_std": 0.0973757952451706,
|
|
"rewards/CSTORM/mean": 0.39453125,
|
|
"rewards/CSTORM/std": 0.20478858053684235,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1314,
|
|
"train_speed(iter/s)": 0.014035
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 953.0,
|
|
"completions/mean_length": 232.65625,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.7625398666280081,
|
|
"grad_norm": 0.4577855954962949,
|
|
"kl": 0.078369140625,
|
|
"learning_rate": 3.694287381831246e-07,
|
|
"loss": 7.843031926313415e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.73828125,
|
|
"reward_std": 0.1911257952451706,
|
|
"rewards/CSTORM/mean": 0.40234375,
|
|
"rewards/CSTORM/std": 0.19899940490722656,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8359375,
|
|
"rewards/VQAORM/std": 0.371787428855896,
|
|
"step": 1315,
|
|
"train_speed(iter/s)": 0.014041
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 559.0,
|
|
"completions/mean_length": 227.9765625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.7631197448535807,
|
|
"grad_norm": 0.44209866220238847,
|
|
"kl": 0.076904296875,
|
|
"learning_rate": 3.692368321407115e-07,
|
|
"loss": 7.695305976085365e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.13928458094596863,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1316,
|
|
"train_speed(iter/s)": 0.014046
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1185.0,
|
|
"completions/mean_length": 297.53125,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.7636996230791534,
|
|
"grad_norm": 0.43096623010462387,
|
|
"kl": 0.06640625,
|
|
"learning_rate": 3.690448429242473e-07,
|
|
"loss": 6.63337777950801e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.1796875,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1317,
|
|
"train_speed(iter/s)": 0.014051
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 633.0,
|
|
"completions/mean_length": 222.90625,
|
|
"completions/min_length": 71.0,
|
|
"epoch": 0.764279501304726,
|
|
"grad_norm": 0.46641064116236713,
|
|
"kl": 0.077392578125,
|
|
"learning_rate": 3.688527707031831e-07,
|
|
"loss": 7.729146454948932e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.19056488573551178,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1318,
|
|
"train_speed(iter/s)": 0.014057
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 767.0,
|
|
"completions/mean_length": 252.390625,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.7648593795302986,
|
|
"grad_norm": 0.44177468334447595,
|
|
"kl": 0.068359375,
|
|
"learning_rate": 3.68660615647043e-07,
|
|
"loss": 6.841956928838044e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.2067507952451706,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1319,
|
|
"train_speed(iter/s)": 0.014063
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 791.0,
|
|
"completions/mean_length": 248.3359375,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.7654392577558713,
|
|
"grad_norm": 0.5604157135412559,
|
|
"kl": 0.07275390625,
|
|
"learning_rate": 3.684683779254245e-07,
|
|
"loss": 7.264584564836696e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.609375,
|
|
"reward_std": 0.25753000378608704,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1320,
|
|
"train_speed(iter/s)": 0.014069
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 656.0,
|
|
"completions/mean_length": 223.953125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.7660191359814439,
|
|
"grad_norm": 0.6209667650042173,
|
|
"kl": 0.072509765625,
|
|
"learning_rate": 3.6827605770799763e-07,
|
|
"loss": 7.264565647346899e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1321,
|
|
"train_speed(iter/s)": 0.014075
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 623.0,
|
|
"completions/mean_length": 214.78125,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.7665990142070165,
|
|
"grad_norm": 0.5890825641522526,
|
|
"kl": 0.07763671875,
|
|
"learning_rate": 3.680836551645057e-07,
|
|
"loss": 7.756828563287854e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62890625,
|
|
"reward_std": 0.2313968986272812,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1322,
|
|
"train_speed(iter/s)": 0.014082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 808.0,
|
|
"completions/mean_length": 251.2734375,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.7671788924325892,
|
|
"grad_norm": 0.4125689970293665,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 3.6789117046476433e-07,
|
|
"loss": 7.002997153904289e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.1400640904903412,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1323,
|
|
"train_speed(iter/s)": 0.014088
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 753.0,
|
|
"completions/mean_length": 252.140625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.7677587706581618,
|
|
"grad_norm": 0.5368295343801273,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 3.6769860377866194e-07,
|
|
"loss": 6.79420045344159e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.2109375,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1324,
|
|
"train_speed(iter/s)": 0.014094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 598.0,
|
|
"completions/mean_length": 236.2578125,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.7683386488837344,
|
|
"grad_norm": 0.5848633693013999,
|
|
"kl": 0.07861328125,
|
|
"learning_rate": 3.67505955276159e-07,
|
|
"loss": 7.854834257159382e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.291347861289978,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1325,
|
|
"train_speed(iter/s)": 0.014101
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 767.0,
|
|
"completions/mean_length": 224.890625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.7689185271093071,
|
|
"grad_norm": 0.6196502332654104,
|
|
"kl": 0.155517578125,
|
|
"learning_rate": 3.6731322512728843e-07,
|
|
"loss": 0.00015526125207543373,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.2751619815826416,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1326,
|
|
"train_speed(iter/s)": 0.014107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 498.0,
|
|
"completions/mean_length": 218.7265625,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.7694984053348797,
|
|
"grad_norm": 0.5339140499325238,
|
|
"kl": 0.0703125,
|
|
"learning_rate": 3.67120413502155e-07,
|
|
"loss": 7.027300307527184e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2301882952451706,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1327,
|
|
"train_speed(iter/s)": 0.014114
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 684.0,
|
|
"completions/mean_length": 231.1484375,
|
|
"completions/min_length": 46.0,
|
|
"epoch": 0.7700782835604523,
|
|
"grad_norm": 0.567873538892742,
|
|
"kl": 0.07421875,
|
|
"learning_rate": 3.669275205709358e-07,
|
|
"loss": 7.430709956679493e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.289578378200531,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1328,
|
|
"train_speed(iter/s)": 0.014119
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 660.0,
|
|
"completions/mean_length": 254.890625,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.770658161786025,
|
|
"grad_norm": 0.580156350646013,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 3.6673454650387923e-07,
|
|
"loss": 6.901874439790845e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.3043861389160156,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1329,
|
|
"train_speed(iter/s)": 0.014125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 545.0,
|
|
"completions/mean_length": 228.3359375,
|
|
"completions/min_length": 64.0,
|
|
"epoch": 0.7712380400115976,
|
|
"grad_norm": 0.45673225729088995,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 3.665414914713055e-07,
|
|
"loss": 7.509491115342826e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.1875,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1330,
|
|
"train_speed(iter/s)": 0.014132
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 528.0,
|
|
"completions/mean_length": 217.6015625,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.7718179182371702,
|
|
"grad_norm": 0.47818592624202994,
|
|
"kl": 0.075439453125,
|
|
"learning_rate": 3.6634835564360624e-07,
|
|
"loss": 7.534719770774245e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.671875,
|
|
"reward_std": 0.1326618194580078,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1331,
|
|
"train_speed(iter/s)": 0.014138
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 659.0,
|
|
"completions/mean_length": 252.171875,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.7723977964627429,
|
|
"grad_norm": 0.5196169450062111,
|
|
"kl": 0.067138671875,
|
|
"learning_rate": 3.6615513919124436e-07,
|
|
"loss": 6.704382394673303e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.54296875,
|
|
"reward_std": 0.24831727147102356,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1332,
|
|
"train_speed(iter/s)": 0.014144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 635.0,
|
|
"completions/mean_length": 222.0234375,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.7729776746883155,
|
|
"grad_norm": 0.37262004614758715,
|
|
"kl": 0.07421875,
|
|
"learning_rate": 3.6596184228475415e-07,
|
|
"loss": 7.4176728958264e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.08578681945800781,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1333,
|
|
"train_speed(iter/s)": 0.014149
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1531.0,
|
|
"completions/mean_length": 241.375,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.7735575529138881,
|
|
"grad_norm": 0.5973402391919668,
|
|
"kl": 0.077392578125,
|
|
"learning_rate": 3.6576846509474055e-07,
|
|
"loss": 7.741889567114413e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.3041265904903412,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1334,
|
|
"train_speed(iter/s)": 0.014144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 889.0,
|
|
"completions/mean_length": 258.484375,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.7741374311394608,
|
|
"grad_norm": 0.42284990652037086,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 3.6557500779187963e-07,
|
|
"loss": 6.880389992147684e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66015625,
|
|
"reward_std": 0.13764688372612,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 1335,
|
|
"train_speed(iter/s)": 0.01415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 968.0,
|
|
"completions/mean_length": 263.9140625,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.7747173093650334,
|
|
"grad_norm": 0.427595433960562,
|
|
"kl": 0.15234375,
|
|
"learning_rate": 3.6538147054691815e-07,
|
|
"loss": 0.00015190521662589163,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.14452920854091644,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1336,
|
|
"train_speed(iter/s)": 0.014105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 824.0,
|
|
"completions/mean_length": 243.125,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.775297187590606,
|
|
"grad_norm": 0.5447017211618684,
|
|
"kl": 0.073974609375,
|
|
"learning_rate": 3.6518785353067323e-07,
|
|
"loss": 7.390962855424732e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.2686898708343506,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1337,
|
|
"train_speed(iter/s)": 0.014111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 903.0,
|
|
"completions/mean_length": 262.359375,
|
|
"completions/min_length": 112.0,
|
|
"epoch": 0.7758770658161787,
|
|
"grad_norm": 0.33228576640888163,
|
|
"kl": 0.071533203125,
|
|
"learning_rate": 3.649941569140326e-07,
|
|
"loss": 7.138633372960612e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.7890625,
|
|
"reward_std": 0.11407758295536041,
|
|
"rewards/CSTORM/mean": 0.421875,
|
|
"rewards/CSTORM/std": 0.1822594404220581,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8671875,
|
|
"rewards/VQAORM/std": 0.3407054841518402,
|
|
"step": 1338,
|
|
"train_speed(iter/s)": 0.014117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 503.0,
|
|
"completions/mean_length": 222.78125,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.7764569440417513,
|
|
"grad_norm": 0.6420581742606067,
|
|
"kl": 0.07763671875,
|
|
"learning_rate": 3.648003808679541e-07,
|
|
"loss": 7.761754386592656e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.2791979908943176,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1339,
|
|
"train_speed(iter/s)": 0.014123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 800.0,
|
|
"completions/mean_length": 232.625,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.7770368222673238,
|
|
"grad_norm": 0.600491925480222,
|
|
"kl": 0.1005859375,
|
|
"learning_rate": 3.6460652556346593e-07,
|
|
"loss": 0.0001005555095616728,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.2512606978416443,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1340,
|
|
"train_speed(iter/s)": 0.014077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 946.0,
|
|
"completions/mean_length": 232.2421875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.7776167004928964,
|
|
"grad_norm": 0.546547130093726,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.6441259117166594e-07,
|
|
"loss": 7.308971544262022e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.216848686337471,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1341,
|
|
"train_speed(iter/s)": 0.014083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1051.0,
|
|
"completions/mean_length": 242.609375,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.7781965787184691,
|
|
"grad_norm": 0.5281935851019484,
|
|
"kl": 0.082763671875,
|
|
"learning_rate": 3.642185778637219e-07,
|
|
"loss": 8.284106297651306e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.64453125,
|
|
"reward_std": 0.1711822748184204,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 1342,
|
|
"train_speed(iter/s)": 0.014077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 519.0,
|
|
"completions/mean_length": 207.7265625,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.7787764569440417,
|
|
"grad_norm": 0.6571196284525211,
|
|
"kl": 0.080810546875,
|
|
"learning_rate": 3.640244858108712e-07,
|
|
"loss": 8.095917291939259e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.2686898708343506,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1343,
|
|
"train_speed(iter/s)": 0.014083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 509.0,
|
|
"completions/mean_length": 236.0703125,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.7793563351696143,
|
|
"grad_norm": 0.4754802711483168,
|
|
"kl": 0.10791015625,
|
|
"learning_rate": 3.638303151844209e-07,
|
|
"loss": 0.0001078423229046166,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.20200318098068237,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1344,
|
|
"train_speed(iter/s)": 0.014089
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 509.0,
|
|
"completions/mean_length": 218.8671875,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.779936213395187,
|
|
"grad_norm": 0.7111479944684549,
|
|
"kl": 0.083984375,
|
|
"learning_rate": 3.636360661557471e-07,
|
|
"loss": 8.391176379518583e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40234375,
|
|
"reward_std": 0.319600909948349,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1345,
|
|
"train_speed(iter/s)": 0.014093
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 505.0,
|
|
"completions/mean_length": 225.234375,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.7805160916207596,
|
|
"grad_norm": 0.4389834530586146,
|
|
"kl": 0.077392578125,
|
|
"learning_rate": 3.6344173889629547e-07,
|
|
"loss": 7.737340638414025e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.546875,
|
|
"reward_std": 0.1573079228401184,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1346,
|
|
"train_speed(iter/s)": 0.014099
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 662.0,
|
|
"completions/mean_length": 238.671875,
|
|
"completions/min_length": 109.0,
|
|
"epoch": 0.7810959698463322,
|
|
"grad_norm": 0.5725608958753744,
|
|
"kl": 0.07177734375,
|
|
"learning_rate": 3.6324733357758037e-07,
|
|
"loss": 7.199628453236073e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58203125,
|
|
"reward_std": 0.21577188372612,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1347,
|
|
"train_speed(iter/s)": 0.014105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 592.0,
|
|
"completions/mean_length": 243.515625,
|
|
"completions/min_length": 57.0,
|
|
"epoch": 0.7816758480719049,
|
|
"grad_norm": 0.550698435239484,
|
|
"kl": 0.0693359375,
|
|
"learning_rate": 3.630528503711854e-07,
|
|
"loss": 6.920019222889096e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.2338140904903412,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 1348,
|
|
"train_speed(iter/s)": 0.01411
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 791.0,
|
|
"completions/mean_length": 243.9140625,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.7822557262974775,
|
|
"grad_norm": 0.6363205753396731,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 3.6285828944876244e-07,
|
|
"loss": 6.772223423467949e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.31154364347457886,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1349,
|
|
"train_speed(iter/s)": 0.014117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 513.0,
|
|
"completions/mean_length": 214.90625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.7828356045230501,
|
|
"grad_norm": 0.421777198636195,
|
|
"kl": 0.075439453125,
|
|
"learning_rate": 3.626636509820325e-07,
|
|
"loss": 7.539401121903211e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.1647101789712906,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1350,
|
|
"train_speed(iter/s)": 0.014123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 541.0,
|
|
"completions/mean_length": 230.265625,
|
|
"completions/min_length": 64.0,
|
|
"epoch": 0.7834154827486228,
|
|
"grad_norm": 0.5636080352986899,
|
|
"kl": 0.0732421875,
|
|
"learning_rate": 3.6246893514278476e-07,
|
|
"loss": 7.310824003070593e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5859375,
|
|
"reward_std": 0.23567037284374237,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1351,
|
|
"train_speed(iter/s)": 0.014129
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 842.0,
|
|
"completions/mean_length": 256.1953125,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.7839953609741954,
|
|
"grad_norm": 0.5920891436518326,
|
|
"kl": 0.346923828125,
|
|
"learning_rate": 3.6227414210287656e-07,
|
|
"loss": 0.0003468106151558459,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.328191876411438,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1352,
|
|
"train_speed(iter/s)": 0.014135
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1262.0,
|
|
"completions/mean_length": 237.4921875,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.784575239199768,
|
|
"grad_norm": 0.5772802165682326,
|
|
"kl": 0.0771484375,
|
|
"learning_rate": 3.6207927203423366e-07,
|
|
"loss": 7.709550118306652e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.24447289109230042,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1353,
|
|
"train_speed(iter/s)": 0.014139
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 940.0,
|
|
"completions/mean_length": 236.1640625,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.7851551174253407,
|
|
"grad_norm": 0.5074405105812139,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 3.618843251088496e-07,
|
|
"loss": 6.685833068331704e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.296875,
|
|
"reward_std": 0.20122367143630981,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 1354,
|
|
"train_speed(iter/s)": 0.014144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 768.0,
|
|
"completions/mean_length": 217.59375,
|
|
"completions/min_length": 97.0,
|
|
"epoch": 0.7857349956509133,
|
|
"grad_norm": 0.5801463145943822,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 3.616893014987859e-07,
|
|
"loss": 7.529727008659393e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6171875,
|
|
"reward_std": 0.31267353892326355,
|
|
"rewards/CSTORM/mean": 0.359375,
|
|
"rewards/CSTORM/std": 0.2256879359483719,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1355,
|
|
"train_speed(iter/s)": 0.01415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3197.0,
|
|
"completions/mean_length": 230.1875,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.7863148738764859,
|
|
"grad_norm": 0.6147700278036266,
|
|
"kl": 0.082275390625,
|
|
"learning_rate": 3.6149420137617155e-07,
|
|
"loss": 8.22298534330912e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.31640625,
|
|
"reward_std": 0.2841641902923584,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 1356,
|
|
"train_speed(iter/s)": 0.014093
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 518.0,
|
|
"completions/mean_length": 212.15625,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.7868947521020586,
|
|
"grad_norm": 0.5139562505627667,
|
|
"kl": 0.07275390625,
|
|
"learning_rate": 3.612990249132033e-07,
|
|
"loss": 7.268632180057466e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.2067507952451706,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1357,
|
|
"train_speed(iter/s)": 0.014099
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 721.0,
|
|
"completions/mean_length": 226.1796875,
|
|
"completions/min_length": 66.0,
|
|
"epoch": 0.7874746303276312,
|
|
"grad_norm": 0.7179809683320103,
|
|
"kl": 0.075927734375,
|
|
"learning_rate": 3.611037722821452e-07,
|
|
"loss": 7.605092832818627e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.3682003617286682,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1358,
|
|
"train_speed(iter/s)": 0.014105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 917.0,
|
|
"completions/mean_length": 249.03125,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.7880545085532038,
|
|
"grad_norm": 0.533791929594911,
|
|
"kl": 0.072265625,
|
|
"learning_rate": 3.6090844365532846e-07,
|
|
"loss": 7.234398799482733e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34765625,
|
|
"reward_std": 0.2300376147031784,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 1359,
|
|
"train_speed(iter/s)": 0.01411
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1634.0,
|
|
"completions/mean_length": 258.625,
|
|
"completions/min_length": 60.0,
|
|
"epoch": 0.7886343867787765,
|
|
"grad_norm": 0.6397052603535938,
|
|
"kl": 0.0732421875,
|
|
"learning_rate": 3.607130392051515e-07,
|
|
"loss": 7.314521644730121e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.1875,
|
|
"reward_std": 0.3088291585445404,
|
|
"rewards/CSTORM/mean": 0.2109375,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.4765625,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 1360,
|
|
"train_speed(iter/s)": 0.014113
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2372.0,
|
|
"completions/mean_length": 273.3203125,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.7892142650043491,
|
|
"grad_norm": 0.4815364001848151,
|
|
"kl": 0.06787109375,
|
|
"learning_rate": 3.605175591040794e-07,
|
|
"loss": 6.786258018109947e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.2107868194580078,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1361,
|
|
"train_speed(iter/s)": 0.014112
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 514.0,
|
|
"completions/mean_length": 203.90625,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.7897941432299217,
|
|
"grad_norm": 0.5719187195862269,
|
|
"kl": 0.08544921875,
|
|
"learning_rate": 3.6032200352464453e-07,
|
|
"loss": 8.54053650982678e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.21279378235340118,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1362,
|
|
"train_speed(iter/s)": 0.014118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1573.0,
|
|
"completions/mean_length": 241.2265625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.7903740214554943,
|
|
"grad_norm": 0.4864749594077905,
|
|
"kl": 0.078125,
|
|
"learning_rate": 3.601263726394452e-07,
|
|
"loss": 7.798292790539563e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.22962738573551178,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1363,
|
|
"train_speed(iter/s)": 0.01412
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 652.0,
|
|
"completions/mean_length": 234.21875,
|
|
"completions/min_length": 110.0,
|
|
"epoch": 0.790953899681067,
|
|
"grad_norm": 0.6250861752981195,
|
|
"kl": 0.07958984375,
|
|
"learning_rate": 3.599306666211466e-07,
|
|
"loss": 7.957174238981679e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66796875,
|
|
"reward_std": 0.2765023708343506,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8203125,
|
|
"rewards/VQAORM/std": 0.3854355216026306,
|
|
"step": 1364,
|
|
"train_speed(iter/s)": 0.014126
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 693.0,
|
|
"completions/mean_length": 239.2265625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.7915337779066396,
|
|
"grad_norm": 0.593833697805689,
|
|
"kl": 0.075927734375,
|
|
"learning_rate": 3.597348856424802e-07,
|
|
"loss": 7.582899706903845e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.359375,
|
|
"reward_std": 0.2534750998020172,
|
|
"rewards/CSTORM/mean": 0.25,
|
|
"rewards/CSTORM/std": 0.2509823143482208,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1365,
|
|
"train_speed(iter/s)": 0.014133
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1199.0,
|
|
"completions/mean_length": 258.3828125,
|
|
"completions/min_length": 95.0,
|
|
"epoch": 0.7921136561322122,
|
|
"grad_norm": 0.45362321537285427,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 3.595390298762437e-07,
|
|
"loss": 6.986738299019635e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.2517244815826416,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1366,
|
|
"train_speed(iter/s)": 0.014135
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 735.0,
|
|
"completions/mean_length": 233.015625,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.7926935343577849,
|
|
"grad_norm": 0.4760606407661159,
|
|
"kl": 0.0810546875,
|
|
"learning_rate": 3.5934309949530046e-07,
|
|
"loss": 8.110634371405467e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.23743988573551178,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1367,
|
|
"train_speed(iter/s)": 0.014141
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 832.0,
|
|
"completions/mean_length": 265.96875,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.7932734125833575,
|
|
"grad_norm": 0.7029494065533756,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.5914709467258014e-07,
|
|
"loss": 7.309546344913542e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.3316189646720886,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 1368,
|
|
"train_speed(iter/s)": 0.014147
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1020.0,
|
|
"completions/mean_length": 218.3203125,
|
|
"completions/min_length": 102.0,
|
|
"epoch": 0.7938532908089301,
|
|
"grad_norm": 0.5035509323031211,
|
|
"kl": 0.07421875,
|
|
"learning_rate": 3.589510155810778e-07,
|
|
"loss": 7.431526319123805e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.2824135720729828,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1369,
|
|
"train_speed(iter/s)": 0.014151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 913.0,
|
|
"completions/mean_length": 216.4375,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.7944331690345028,
|
|
"grad_norm": 0.5697012818533336,
|
|
"kl": 0.080078125,
|
|
"learning_rate": 3.5875486239385403e-07,
|
|
"loss": 7.995944179128855e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.21684867143630981,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1370,
|
|
"train_speed(iter/s)": 0.014157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 545.0,
|
|
"completions/mean_length": 195.78125,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.7950130472600754,
|
|
"grad_norm": 0.45138637602482423,
|
|
"kl": 0.08837890625,
|
|
"learning_rate": 3.5855863528403516e-07,
|
|
"loss": 8.820240327622741e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.7265625,
|
|
"reward_std": 0.16997367143630981,
|
|
"rewards/CSTORM/mean": 0.40625,
|
|
"rewards/CSTORM/std": 0.19592301547527313,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8203125,
|
|
"rewards/VQAORM/std": 0.3854355216026306,
|
|
"step": 1371,
|
|
"train_speed(iter/s)": 0.014164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 496.0,
|
|
"completions/mean_length": 209.921875,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.795592925485648,
|
|
"grad_norm": 0.4023884451650001,
|
|
"kl": 0.091064453125,
|
|
"learning_rate": 3.583623344248124e-07,
|
|
"loss": 9.101125760935247e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.28125,
|
|
"reward_std": 0.1875,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5234375,
|
|
"rewards/VQAORM/std": 0.5014128684997559,
|
|
"step": 1372,
|
|
"train_speed(iter/s)": 0.014169
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 741.0,
|
|
"completions/mean_length": 237.796875,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.7961728037112207,
|
|
"grad_norm": 0.5366736956860123,
|
|
"kl": 0.08203125,
|
|
"learning_rate": 3.581659599894422e-07,
|
|
"loss": 8.18571716081351e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.234375,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1373,
|
|
"train_speed(iter/s)": 0.014175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1038.0,
|
|
"completions/mean_length": 254.875,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.7967526819367933,
|
|
"grad_norm": 0.4781482434402123,
|
|
"kl": 0.073974609375,
|
|
"learning_rate": 3.579695121512459e-07,
|
|
"loss": 7.392857514787465e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.73828125,
|
|
"reward_std": 0.19150984287261963,
|
|
"rewards/CSTORM/mean": 0.40234375,
|
|
"rewards/CSTORM/std": 0.19899940490722656,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8359375,
|
|
"rewards/VQAORM/std": 0.371787428855896,
|
|
"step": 1374,
|
|
"train_speed(iter/s)": 0.014178
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 623.0,
|
|
"completions/mean_length": 198.3046875,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.7973325601623659,
|
|
"grad_norm": 0.6191022591754289,
|
|
"kl": 0.081298828125,
|
|
"learning_rate": 3.577729910836098e-07,
|
|
"loss": 8.132416405715048e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.31258678436279297,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1375,
|
|
"train_speed(iter/s)": 0.014184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 540.0,
|
|
"completions/mean_length": 208.8203125,
|
|
"completions/min_length": 59.0,
|
|
"epoch": 0.7979124383879386,
|
|
"grad_norm": 0.555360376050539,
|
|
"kl": 0.08154296875,
|
|
"learning_rate": 3.575763969599845e-07,
|
|
"loss": 8.14093291410245e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.20703125,
|
|
"reward_std": 0.23827922344207764,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.484375,
|
|
"rewards/VQAORM/std": 0.5017194747924805,
|
|
"step": 1376,
|
|
"train_speed(iter/s)": 0.014191
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 520.0,
|
|
"completions/mean_length": 200.609375,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.7984923166135112,
|
|
"grad_norm": 0.5521530281994665,
|
|
"kl": 0.08447265625,
|
|
"learning_rate": 3.5737972995388537e-07,
|
|
"loss": 8.438416989520192e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.2260015904903412,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1377,
|
|
"train_speed(iter/s)": 0.014197
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1019.0,
|
|
"completions/mean_length": 225.8828125,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.7990721948390838,
|
|
"grad_norm": 0.6566208671870388,
|
|
"kl": 0.08056640625,
|
|
"learning_rate": 3.57182990238892e-07,
|
|
"loss": 8.076739322859794e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.2793486714363098,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1378,
|
|
"train_speed(iter/s)": 0.014202
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 606.0,
|
|
"completions/mean_length": 224.2421875,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.7996520730646565,
|
|
"grad_norm": 0.3964550642808601,
|
|
"kl": 0.08642578125,
|
|
"learning_rate": 3.5698617798864815e-07,
|
|
"loss": 8.622118184575811e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.57421875,
|
|
"reward_std": 0.0859375,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1379,
|
|
"train_speed(iter/s)": 0.014208
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1158.0,
|
|
"completions/mean_length": 239.8671875,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.8002319512902291,
|
|
"grad_norm": 0.5331254871496044,
|
|
"kl": 0.076904296875,
|
|
"learning_rate": 3.567892933768617e-07,
|
|
"loss": 7.684497541049495e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.1946197748184204,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.703125,
|
|
"rewards/VQAORM/std": 0.45867621898651123,
|
|
"step": 1380,
|
|
"train_speed(iter/s)": 0.014212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 825.0,
|
|
"completions/mean_length": 245.328125,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.8008118295158017,
|
|
"grad_norm": 0.4472951702267736,
|
|
"kl": 0.072265625,
|
|
"learning_rate": 3.565923365773042e-07,
|
|
"loss": 7.223094871733338e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.17834708094596863,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1381,
|
|
"train_speed(iter/s)": 0.014218
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 717.0,
|
|
"completions/mean_length": 237.5859375,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.8013917077413744,
|
|
"grad_norm": 0.5887346073529306,
|
|
"kl": 0.064453125,
|
|
"learning_rate": 3.563953077638111e-07,
|
|
"loss": 6.450037471950054e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4140625,
|
|
"reward_std": 0.3239383101463318,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1382,
|
|
"train_speed(iter/s)": 0.014224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 667.0,
|
|
"completions/mean_length": 210.28125,
|
|
"completions/min_length": 89.0,
|
|
"epoch": 0.801971585966947,
|
|
"grad_norm": 0.5433660048482587,
|
|
"kl": 0.084716796875,
|
|
"learning_rate": 3.5619820711028124e-07,
|
|
"loss": 8.472017361782491e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6328125,
|
|
"reward_std": 0.2414947748184204,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1383,
|
|
"train_speed(iter/s)": 0.01423
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 546.0,
|
|
"completions/mean_length": 232.828125,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.8025514641925195,
|
|
"grad_norm": 0.5553356592373432,
|
|
"kl": 0.072021484375,
|
|
"learning_rate": 3.560010347906771e-07,
|
|
"loss": 7.19842646503821e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6953125,
|
|
"reward_std": 0.190433070063591,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8125,
|
|
"rewards/VQAORM/std": 0.39184603095054626,
|
|
"step": 1384,
|
|
"train_speed(iter/s)": 0.014236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 554.0,
|
|
"completions/mean_length": 217.421875,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.8031313424180923,
|
|
"grad_norm": 0.6817100937876054,
|
|
"kl": 0.0810546875,
|
|
"learning_rate": 3.558037909790241e-07,
|
|
"loss": 8.118960977299139e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51171875,
|
|
"reward_std": 0.3527260720729828,
|
|
"rewards/CSTORM/mean": 0.33203125,
|
|
"rewards/CSTORM/std": 0.23708651959896088,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1385,
|
|
"train_speed(iter/s)": 0.014243
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 546.0,
|
|
"completions/mean_length": 211.0234375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.8037112206436648,
|
|
"grad_norm": 0.5267852912027435,
|
|
"kl": 0.08203125,
|
|
"learning_rate": 3.5560647584941117e-07,
|
|
"loss": 8.187373168766499e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.2348908632993698,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1386,
|
|
"train_speed(iter/s)": 0.014249
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 504.0,
|
|
"completions/mean_length": 212.203125,
|
|
"completions/min_length": 60.0,
|
|
"epoch": 0.8042910988692374,
|
|
"grad_norm": 0.5420326901121267,
|
|
"kl": 0.081787109375,
|
|
"learning_rate": 3.5540908957598956e-07,
|
|
"loss": 8.175274706445634e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.69140625,
|
|
"reward_std": 0.18275238573551178,
|
|
"rewards/CSTORM/mean": 0.38671875,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8046875,
|
|
"rewards/VQAORM/std": 0.3979988098144531,
|
|
"step": 1387,
|
|
"train_speed(iter/s)": 0.014244
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 514.0,
|
|
"completions/mean_length": 218.640625,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.80487097709481,
|
|
"grad_norm": 0.6949154909218858,
|
|
"kl": 0.078369140625,
|
|
"learning_rate": 3.5521163233297416e-07,
|
|
"loss": 7.83278519520536e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.3666265904903412,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1388,
|
|
"train_speed(iter/s)": 0.01425
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 710.0,
|
|
"completions/mean_length": 225.8125,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.8054508553203827,
|
|
"grad_norm": 0.5350394030948988,
|
|
"kl": 0.076904296875,
|
|
"learning_rate": 3.5501410429464175e-07,
|
|
"loss": 7.695043314015493e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.36328125,
|
|
"reward_std": 0.2806890904903412,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.578125,
|
|
"rewards/VQAORM/std": 0.4957992732524872,
|
|
"step": 1389,
|
|
"train_speed(iter/s)": 0.014257
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 518.0,
|
|
"completions/mean_length": 218.2109375,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.8060307335459553,
|
|
"grad_norm": 0.5312790262394873,
|
|
"kl": 0.087646484375,
|
|
"learning_rate": 3.5481650563533196e-07,
|
|
"loss": 8.761802746448666e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.22265625,
|
|
"reward_std": 0.1723720133304596,
|
|
"rewards/CSTORM/mean": 0.22265625,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5,
|
|
"rewards/VQAORM/std": 0.5019646286964417,
|
|
"step": 1390,
|
|
"train_speed(iter/s)": 0.014262
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1118.0,
|
|
"completions/mean_length": 263.53125,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.8066106117715279,
|
|
"grad_norm": 0.4143902706699091,
|
|
"kl": 0.068359375,
|
|
"learning_rate": 3.546188365294466e-07,
|
|
"loss": 6.848929479019716e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.1791265904903412,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1391,
|
|
"train_speed(iter/s)": 0.014267
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 484.0,
|
|
"completions/mean_length": 198.3828125,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.8071904899971006,
|
|
"grad_norm": 0.5976398655912071,
|
|
"kl": 0.087158203125,
|
|
"learning_rate": 3.5442109715144984e-07,
|
|
"loss": 8.704322681296617e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.2144126147031784,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1392,
|
|
"train_speed(iter/s)": 0.014274
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 912.0,
|
|
"completions/mean_length": 260.9140625,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.8077703682226732,
|
|
"grad_norm": 0.6208893322930237,
|
|
"kl": 0.06982421875,
|
|
"learning_rate": 3.542232876758677e-07,
|
|
"loss": 6.984520587138832e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.327413409948349,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1393,
|
|
"train_speed(iter/s)": 0.014279
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1023.0,
|
|
"completions/mean_length": 236.8984375,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.8083502464482458,
|
|
"grad_norm": 0.6665888343601107,
|
|
"kl": 0.07080078125,
|
|
"learning_rate": 3.540254082772882e-07,
|
|
"loss": 7.08164443494752e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.289578378200531,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1394,
|
|
"train_speed(iter/s)": 0.014284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 905.0,
|
|
"completions/mean_length": 230.34375,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.8089301246738185,
|
|
"grad_norm": 0.5643807979005604,
|
|
"kl": 0.068603515625,
|
|
"learning_rate": 3.5382745913036094e-07,
|
|
"loss": 6.859630957478657e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.218038409948349,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1395,
|
|
"train_speed(iter/s)": 0.014289
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 839.0,
|
|
"completions/mean_length": 234.078125,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.8095100028993911,
|
|
"grad_norm": 0.6174842808663664,
|
|
"kl": 0.076904296875,
|
|
"learning_rate": 3.5362944040979714e-07,
|
|
"loss": 7.690068741794676e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.328125,
|
|
"reward_std": 0.2312462031841278,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 1396,
|
|
"train_speed(iter/s)": 0.014294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 601.0,
|
|
"completions/mean_length": 228.21875,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.8100898811249637,
|
|
"grad_norm": 0.47880803686717804,
|
|
"kl": 0.082275390625,
|
|
"learning_rate": 3.534313522903694e-07,
|
|
"loss": 8.228512160712853e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.640625,
|
|
"reward_std": 0.1501619666814804,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1397,
|
|
"train_speed(iter/s)": 0.0143
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1855.0,
|
|
"completions/mean_length": 260.1640625,
|
|
"completions/min_length": 62.0,
|
|
"epoch": 0.8106697593505364,
|
|
"grad_norm": 0.4769734527280176,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.532331949469117e-07,
|
|
"loss": 7.279480632860214e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62890625,
|
|
"reward_std": 0.2643525004386902,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1398,
|
|
"train_speed(iter/s)": 0.014301
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1003.0,
|
|
"completions/mean_length": 257.2265625,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.811249637576109,
|
|
"grad_norm": 0.45855111700501416,
|
|
"kl": 0.064697265625,
|
|
"learning_rate": 3.5303496855431895e-07,
|
|
"loss": 6.47461783955805e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.21762818098068237,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1399,
|
|
"train_speed(iter/s)": 0.014306
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 679.0,
|
|
"completions/mean_length": 231.0078125,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.8118295158016816,
|
|
"grad_norm": 0.5337151476304495,
|
|
"kl": 0.0810546875,
|
|
"learning_rate": 3.528366732875471e-07,
|
|
"loss": 8.109876944217831e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.51953125,
|
|
"reward_std": 0.17614847421646118,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1400,
|
|
"train_speed(iter/s)": 0.014311
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 558.0,
|
|
"completions/mean_length": 217.8359375,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.8124093940272543,
|
|
"grad_norm": 0.5596279485835918,
|
|
"kl": 0.079345703125,
|
|
"learning_rate": 3.526383093216129e-07,
|
|
"loss": 7.945973629830405e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.2559111714363098,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 1401,
|
|
"train_speed(iter/s)": 0.014313
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1069.0,
|
|
"completions/mean_length": 245.25,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.8129892722528269,
|
|
"grad_norm": 0.6204258405500647,
|
|
"kl": 0.071533203125,
|
|
"learning_rate": 3.5243987683159373e-07,
|
|
"loss": 7.138856744859368e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.515625,
|
|
"reward_std": 0.2957531809806824,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1402,
|
|
"train_speed(iter/s)": 0.014318
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 522.0,
|
|
"completions/mean_length": 217.6171875,
|
|
"completions/min_length": 63.0,
|
|
"epoch": 0.8135691504783995,
|
|
"grad_norm": 0.4090599369420903,
|
|
"kl": 0.085693359375,
|
|
"learning_rate": 3.522413759926272e-07,
|
|
"loss": 8.573519880883396e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.1484375,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7421875,
|
|
"rewards/VQAORM/std": 0.43914902210235596,
|
|
"step": 1403,
|
|
"train_speed(iter/s)": 0.014324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 519.0,
|
|
"completions/mean_length": 207.4921875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.8141490287039722,
|
|
"grad_norm": 0.5375066262002551,
|
|
"kl": 0.08447265625,
|
|
"learning_rate": 3.5204280697991165e-07,
|
|
"loss": 8.459292439511046e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.71484375,
|
|
"reward_std": 0.212643101811409,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8359375,
|
|
"rewards/VQAORM/std": 0.371787428855896,
|
|
"step": 1404,
|
|
"train_speed(iter/s)": 0.01433
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 699.0,
|
|
"completions/mean_length": 225.1328125,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.8147289069295448,
|
|
"grad_norm": 0.47748715937140196,
|
|
"kl": 0.079345703125,
|
|
"learning_rate": 3.5184416996870535e-07,
|
|
"loss": 7.934935274533927e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.61328125,
|
|
"reward_std": 0.17196176946163177,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1405,
|
|
"train_speed(iter/s)": 0.014336
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 751.0,
|
|
"completions/mean_length": 214.921875,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.8153087851551174,
|
|
"grad_norm": 0.5507826564400343,
|
|
"kl": 0.082275390625,
|
|
"learning_rate": 3.516454651343267e-07,
|
|
"loss": 8.227373473346233e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.230056494474411,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1406,
|
|
"train_speed(iter/s)": 0.014342
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 523.0,
|
|
"completions/mean_length": 212.1796875,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.8158886633806901,
|
|
"grad_norm": 0.5913403237772717,
|
|
"kl": 0.077880859375,
|
|
"learning_rate": 3.514466926521538e-07,
|
|
"loss": 7.766204362269491e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.421875,
|
|
"reward_std": 0.2777109742164612,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1407,
|
|
"train_speed(iter/s)": 0.014347
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 637.0,
|
|
"completions/mean_length": 205.6875,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.8164685416062627,
|
|
"grad_norm": 0.5371470564437104,
|
|
"kl": 0.076904296875,
|
|
"learning_rate": 3.512478526976247e-07,
|
|
"loss": 7.691615610383451e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.17657756805419922,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1408,
|
|
"train_speed(iter/s)": 0.014353
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 499.0,
|
|
"completions/mean_length": 200.2109375,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.8170484198318353,
|
|
"grad_norm": 0.4119660428170691,
|
|
"kl": 0.08984375,
|
|
"learning_rate": 3.510489454462367e-07,
|
|
"loss": 8.993761730380356e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.76953125,
|
|
"reward_std": 0.1286257952451706,
|
|
"rewards/CSTORM/mean": 0.41796875,
|
|
"rewards/CSTORM/std": 0.185893714427948,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8515625,
|
|
"rewards/VQAORM/std": 0.356930136680603,
|
|
"step": 1409,
|
|
"train_speed(iter/s)": 0.01436
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 710.0,
|
|
"completions/mean_length": 221.8515625,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.817628298057408,
|
|
"grad_norm": 0.4924797516441762,
|
|
"kl": 0.078125,
|
|
"learning_rate": 3.508499710735467e-07,
|
|
"loss": 7.811688556103036e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4296875,
|
|
"reward_std": 0.1922025829553604,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1410,
|
|
"train_speed(iter/s)": 0.014366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 630.0,
|
|
"completions/mean_length": 222.0546875,
|
|
"completions/min_length": 49.0,
|
|
"epoch": 0.8182081762829806,
|
|
"grad_norm": 0.3751181908689435,
|
|
"kl": 0.07861328125,
|
|
"learning_rate": 3.5065092975517086e-07,
|
|
"loss": 7.861181802581996e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.47265625,
|
|
"reward_std": 0.13587738573551178,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1411,
|
|
"train_speed(iter/s)": 0.014372
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 709.0,
|
|
"completions/mean_length": 215.953125,
|
|
"completions/min_length": 49.0,
|
|
"epoch": 0.8187880545085532,
|
|
"grad_norm": 0.5652527136372867,
|
|
"kl": 0.081787109375,
|
|
"learning_rate": 3.504518216667843e-07,
|
|
"loss": 8.163464372046292e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.23531997203826904,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1412,
|
|
"train_speed(iter/s)": 0.014378
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 657.0,
|
|
"completions/mean_length": 227.1171875,
|
|
"completions/min_length": 66.0,
|
|
"epoch": 0.8193679327341258,
|
|
"grad_norm": 0.5343027664830488,
|
|
"kl": 0.0751953125,
|
|
"learning_rate": 3.502526469841212e-07,
|
|
"loss": 7.500371430069208e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.65625,
|
|
"reward_std": 0.17735707759857178,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1413,
|
|
"train_speed(iter/s)": 0.014384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 677.0,
|
|
"completions/mean_length": 241.3828125,
|
|
"completions/min_length": 101.0,
|
|
"epoch": 0.8199478109596985,
|
|
"grad_norm": 0.6505448785092232,
|
|
"kl": 0.07470703125,
|
|
"learning_rate": 3.5005340588297436e-07,
|
|
"loss": 7.476894825231284e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.3088291883468628,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1414,
|
|
"train_speed(iter/s)": 0.01439
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1054.0,
|
|
"completions/mean_length": 223.65625,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.8205276891852711,
|
|
"grad_norm": 0.6596279242533424,
|
|
"kl": 0.07275390625,
|
|
"learning_rate": 3.498540985391954e-07,
|
|
"loss": 7.288010965567082e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66796875,
|
|
"reward_std": 0.23139688372612,
|
|
"rewards/CSTORM/mean": 0.37109375,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 1415,
|
|
"train_speed(iter/s)": 0.014394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 954.0,
|
|
"completions/mean_length": 249.734375,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.8211075674108437,
|
|
"grad_norm": 0.46438132887264766,
|
|
"kl": 0.069091796875,
|
|
"learning_rate": 3.496547251286942e-07,
|
|
"loss": 6.90035376464948e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.19056488573551178,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1416,
|
|
"train_speed(iter/s)": 0.0144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 529.0,
|
|
"completions/mean_length": 213.3359375,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.8216874456364164,
|
|
"grad_norm": 0.6153136256657461,
|
|
"kl": 0.072021484375,
|
|
"learning_rate": 3.494552858274391e-07,
|
|
"loss": 7.204424764495343e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.2723156809806824,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1417,
|
|
"train_speed(iter/s)": 0.014406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1217.0,
|
|
"completions/mean_length": 257.1484375,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.822267323861989,
|
|
"grad_norm": 0.4831852010911127,
|
|
"kl": 0.074462890625,
|
|
"learning_rate": 3.492557808114565e-07,
|
|
"loss": 7.438583270413801e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.24809867143630981,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1418,
|
|
"train_speed(iter/s)": 0.01441
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 652.0,
|
|
"completions/mean_length": 241.859375,
|
|
"completions/min_length": 98.0,
|
|
"epoch": 0.8228472020875616,
|
|
"grad_norm": 0.5998113381736767,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 3.490562102568309e-07,
|
|
"loss": 7.051063585095108e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.2521347105503082,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1419,
|
|
"train_speed(iter/s)": 0.014403
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1155.0,
|
|
"completions/mean_length": 263.7890625,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.8234270803131343,
|
|
"grad_norm": 0.4305372013010527,
|
|
"kl": 0.073974609375,
|
|
"learning_rate": 3.488565743397046e-07,
|
|
"loss": 7.409010140690953e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41015625,
|
|
"reward_std": 0.1555572748184204,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1420,
|
|
"train_speed(iter/s)": 0.014396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 594.0,
|
|
"completions/mean_length": 230.75,
|
|
"completions/min_length": 108.0,
|
|
"epoch": 0.8240069585387069,
|
|
"grad_norm": 0.4685249107509964,
|
|
"kl": 0.07177734375,
|
|
"learning_rate": 3.486568732362775e-07,
|
|
"loss": 7.17782968422398e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.18615958094596863,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1421,
|
|
"train_speed(iter/s)": 0.014402
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 605.0,
|
|
"completions/mean_length": 210.734375,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.8245868367642795,
|
|
"grad_norm": 0.5027893285639959,
|
|
"kl": 0.079833984375,
|
|
"learning_rate": 3.484571071228073e-07,
|
|
"loss": 7.986996206454933e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.55859375,
|
|
"reward_std": 0.19975145161151886,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1422,
|
|
"train_speed(iter/s)": 0.014408
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 612.0,
|
|
"completions/mean_length": 236.484375,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.8251667149898522,
|
|
"grad_norm": 0.5209183546483216,
|
|
"kl": 0.0732421875,
|
|
"learning_rate": 3.482572761756086e-07,
|
|
"loss": 7.324805483222008e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44140625,
|
|
"reward_std": 0.2067507952451706,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1423,
|
|
"train_speed(iter/s)": 0.014414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 535.0,
|
|
"completions/mean_length": 206.0859375,
|
|
"completions/min_length": 66.0,
|
|
"epoch": 0.8257465932154248,
|
|
"grad_norm": 0.5018248578990464,
|
|
"kl": 0.080322265625,
|
|
"learning_rate": 3.4805738057105376e-07,
|
|
"loss": 8.017633808776736e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.56640625,
|
|
"reward_std": 0.174247145652771,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1424,
|
|
"train_speed(iter/s)": 0.014421
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 876.0,
|
|
"completions/mean_length": 232.703125,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.8263264714409974,
|
|
"grad_norm": 0.5974211653905275,
|
|
"kl": 0.072021484375,
|
|
"learning_rate": 3.4785742048557186e-07,
|
|
"loss": 7.212234777398407e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.33203125,
|
|
"reward_std": 0.2380007952451706,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 1425,
|
|
"train_speed(iter/s)": 0.014414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1150.0,
|
|
"completions/mean_length": 219.3359375,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.8269063496665701,
|
|
"grad_norm": 0.6071793058693891,
|
|
"kl": 0.085693359375,
|
|
"learning_rate": 3.4765739609564903e-07,
|
|
"loss": 8.588494529249147e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6015625,
|
|
"reward_std": 0.2957531809806824,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1426,
|
|
"train_speed(iter/s)": 0.014418
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1695.0,
|
|
"completions/mean_length": 262.015625,
|
|
"completions/min_length": 68.0,
|
|
"epoch": 0.8274862278921427,
|
|
"grad_norm": 0.5279645649611767,
|
|
"kl": 0.071044921875,
|
|
"learning_rate": 3.474573075778281e-07,
|
|
"loss": 7.10931490175426e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.23516927659511566,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1427,
|
|
"train_speed(iter/s)": 0.01442
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 690.0,
|
|
"completions/mean_length": 218.5234375,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.8280661061177153,
|
|
"grad_norm": 0.5905453887516452,
|
|
"kl": 0.080078125,
|
|
"learning_rate": 3.4725715510870853e-07,
|
|
"loss": 7.994713087100536e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6640625,
|
|
"reward_std": 0.22332076728343964,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 1428,
|
|
"train_speed(iter/s)": 0.014426
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 582.0,
|
|
"completions/mean_length": 227.7421875,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.828645984343288,
|
|
"grad_norm": 0.46889843200199066,
|
|
"kl": 0.0771484375,
|
|
"learning_rate": 3.4705693886494615e-07,
|
|
"loss": 7.699590059928596e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.16257140040397644,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1429,
|
|
"train_speed(iter/s)": 0.014431
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 718.0,
|
|
"completions/mean_length": 229.390625,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.8292258625688606,
|
|
"grad_norm": 0.5232120513897778,
|
|
"kl": 0.076416015625,
|
|
"learning_rate": 3.468566590232532e-07,
|
|
"loss": 7.64317883295007e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.19716878235340118,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1430,
|
|
"train_speed(iter/s)": 0.014437
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 620.0,
|
|
"completions/mean_length": 236.9375,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.8298057407944331,
|
|
"grad_norm": 0.548533030396774,
|
|
"kl": 0.071533203125,
|
|
"learning_rate": 3.46656315760398e-07,
|
|
"loss": 7.151543104555458e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48828125,
|
|
"reward_std": 0.2607455849647522,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1431,
|
|
"train_speed(iter/s)": 0.014443
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 864.0,
|
|
"completions/mean_length": 228.8984375,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.8303856190200058,
|
|
"grad_norm": 0.5068721967035819,
|
|
"kl": 0.076171875,
|
|
"learning_rate": 3.464559092532048e-07,
|
|
"loss": 7.61387636885047e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62890625,
|
|
"reward_std": 0.1779179871082306,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.78125,
|
|
"rewards/VQAORM/std": 0.41502299904823303,
|
|
"step": 1432,
|
|
"train_speed(iter/s)": 0.014449
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 636.0,
|
|
"completions/mean_length": 233.953125,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.8309654972455784,
|
|
"grad_norm": 0.5455857232880743,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.4625543967855376e-07,
|
|
"loss": 7.286724576260895e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.50390625,
|
|
"reward_std": 0.23623128235340118,
|
|
"rewards/CSTORM/mean": 0.32421875,
|
|
"rewards/CSTORM/std": 0.2396671175956726,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1433,
|
|
"train_speed(iter/s)": 0.014455
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 566.0,
|
|
"completions/mean_length": 231.40625,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.831545375471151,
|
|
"grad_norm": 0.45585099269564694,
|
|
"kl": 0.08447265625,
|
|
"learning_rate": 3.460549072133806e-07,
|
|
"loss": 8.443721162620932e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.37109375,
|
|
"reward_std": 0.1755007952451706,
|
|
"rewards/CSTORM/mean": 0.27734375,
|
|
"rewards/CSTORM/std": 0.2494765669107437,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1434,
|
|
"train_speed(iter/s)": 0.014461
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 631.0,
|
|
"completions/mean_length": 207.34375,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.8321252536967237,
|
|
"grad_norm": 0.6316974050087742,
|
|
"kl": 0.088623046875,
|
|
"learning_rate": 3.4585431203467664e-07,
|
|
"loss": 8.861014794092625e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.72265625,
|
|
"reward_std": 0.3077523708343506,
|
|
"rewards/CSTORM/mean": 0.40234375,
|
|
"rewards/CSTORM/std": 0.19899940490722656,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8203125,
|
|
"rewards/VQAORM/std": 0.3854355216026306,
|
|
"step": 1435,
|
|
"train_speed(iter/s)": 0.014467
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1019.0,
|
|
"completions/mean_length": 232.921875,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.8327051319222963,
|
|
"grad_norm": 0.5475410177186897,
|
|
"kl": 0.074951171875,
|
|
"learning_rate": 3.4565365431948843e-07,
|
|
"loss": 7.485634705517441e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.53515625,
|
|
"reward_std": 0.20014688372612,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1436,
|
|
"train_speed(iter/s)": 0.014472
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1241.0,
|
|
"completions/mean_length": 232.359375,
|
|
"completions/min_length": 106.0,
|
|
"epoch": 0.8332850101478689,
|
|
"grad_norm": 0.47515393980005693,
|
|
"kl": 0.071044921875,
|
|
"learning_rate": 3.4545293424491795e-07,
|
|
"loss": 7.107813144102693e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.640625,
|
|
"reward_std": 0.21762818098068237,
|
|
"rewards/CSTORM/mean": 0.375,
|
|
"rewards/CSTORM/std": 0.2173570692539215,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.765625,
|
|
"rewards/VQAORM/std": 0.42527204751968384,
|
|
"step": 1437,
|
|
"train_speed(iter/s)": 0.014476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 640.0,
|
|
"completions/mean_length": 213.6171875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.8338648883734415,
|
|
"grad_norm": 0.4292400016978,
|
|
"kl": 0.0810546875,
|
|
"learning_rate": 3.4525215198812187e-07,
|
|
"loss": 8.095078374026343e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.14545938372612,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1438,
|
|
"train_speed(iter/s)": 0.014482
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 530.0,
|
|
"completions/mean_length": 230.2265625,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.8344447665990142,
|
|
"grad_norm": 0.5715908423568425,
|
|
"kl": 0.07568359375,
|
|
"learning_rate": 3.4505130772631213e-07,
|
|
"loss": 7.577614451292902e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.34375,
|
|
"reward_std": 0.2614383101463318,
|
|
"rewards/CSTORM/mean": 0.2734375,
|
|
"rewards/CSTORM/std": 0.2498769313097,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5703125,
|
|
"rewards/VQAORM/std": 0.4969765841960907,
|
|
"step": 1439,
|
|
"train_speed(iter/s)": 0.014487
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 708.0,
|
|
"completions/mean_length": 233.2265625,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.8350246448245868,
|
|
"grad_norm": 0.47328455971007183,
|
|
"kl": 0.07958984375,
|
|
"learning_rate": 3.448504016367551e-07,
|
|
"loss": 7.956440094858408e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.67578125,
|
|
"reward_std": 0.22721019387245178,
|
|
"rewards/CSTORM/mean": 0.38671875,
|
|
"rewards/CSTORM/std": 0.21012598276138306,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1440,
|
|
"train_speed(iter/s)": 0.014493
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 667.0,
|
|
"completions/mean_length": 214.125,
|
|
"completions/min_length": 57.0,
|
|
"epoch": 0.8356045230501594,
|
|
"grad_norm": 0.43759260454592985,
|
|
"kl": 0.08251953125,
|
|
"learning_rate": 3.446494338967718e-07,
|
|
"loss": 8.240866736741737e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.66015625,
|
|
"reward_std": 0.13091117143630981,
|
|
"rewards/CSTORM/mean": 0.37109375,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1441,
|
|
"train_speed(iter/s)": 0.014498
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1237.0,
|
|
"completions/mean_length": 239.6953125,
|
|
"completions/min_length": 114.0,
|
|
"epoch": 0.8361844012757321,
|
|
"grad_norm": 0.5386447034058178,
|
|
"kl": 0.07421875,
|
|
"learning_rate": 3.444484046837378e-07,
|
|
"loss": 7.439132605213672e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5,
|
|
"reward_std": 0.2416265904903412,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1442,
|
|
"train_speed(iter/s)": 0.014502
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 963.0,
|
|
"completions/mean_length": 238.0546875,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.8367642795013047,
|
|
"grad_norm": 0.47301290961315845,
|
|
"kl": 0.07080078125,
|
|
"learning_rate": 3.442473141750826e-07,
|
|
"loss": 7.074276800267398e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.18318147957324982,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1443,
|
|
"train_speed(iter/s)": 0.014506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 692.0,
|
|
"completions/mean_length": 212.6015625,
|
|
"completions/min_length": 59.0,
|
|
"epoch": 0.8373441577268773,
|
|
"grad_norm": 0.4714895411555914,
|
|
"kl": 0.0791015625,
|
|
"learning_rate": 3.440461625482902e-07,
|
|
"loss": 7.898636977188289e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.125515878200531,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.640625,
|
|
"rewards/VQAORM/std": 0.481702595949173,
|
|
"step": 1444,
|
|
"train_speed(iter/s)": 0.014512
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 572.0,
|
|
"completions/mean_length": 225.359375,
|
|
"completions/min_length": 86.0,
|
|
"epoch": 0.83792403595245,
|
|
"grad_norm": 0.5845083255333473,
|
|
"kl": 0.074462890625,
|
|
"learning_rate": 3.438449499808983e-07,
|
|
"loss": 7.435933366650715e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.32421875,
|
|
"reward_std": 0.24447289109230042,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5546875,
|
|
"rewards/VQAORM/std": 0.4989531338214874,
|
|
"step": 1445,
|
|
"train_speed(iter/s)": 0.014517
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 560.0,
|
|
"completions/mean_length": 205.1640625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.8385039141780226,
|
|
"grad_norm": 0.4479102937040053,
|
|
"kl": 0.08056640625,
|
|
"learning_rate": 3.436436766504985e-07,
|
|
"loss": 8.060104300966486e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.49609375,
|
|
"reward_std": 0.1730836033821106,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6796875,
|
|
"rewards/VQAORM/std": 0.4684300124645233,
|
|
"step": 1446,
|
|
"train_speed(iter/s)": 0.014522
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1525.0,
|
|
"completions/mean_length": 261.7734375,
|
|
"completions/min_length": 70.0,
|
|
"epoch": 0.8390837924035952,
|
|
"grad_norm": 0.5032393893012581,
|
|
"kl": 0.075927734375,
|
|
"learning_rate": 3.434423427347359e-07,
|
|
"loss": 7.57578673074022e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.375,
|
|
"reward_std": 0.2301882952451706,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1447,
|
|
"train_speed(iter/s)": 0.014525
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 846.0,
|
|
"completions/mean_length": 222.9921875,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.8396636706291679,
|
|
"grad_norm": 0.5159266343314988,
|
|
"kl": 0.08203125,
|
|
"learning_rate": 3.432409484113091e-07,
|
|
"loss": 8.21588619146496e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.20795938372612,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1448,
|
|
"train_speed(iter/s)": 0.01453
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 498.0,
|
|
"completions/mean_length": 198.2421875,
|
|
"completions/min_length": 1.0,
|
|
"epoch": 0.8402435488547405,
|
|
"grad_norm": 23015.887996768288,
|
|
"kl": 25600.04150390625,
|
|
"learning_rate": 3.430394938579701e-07,
|
|
"loss": 25.625080108642578,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.1764625608921051,
|
|
"rewards/CSTORM/mean": 0.34765625,
|
|
"rewards/CSTORM/std": 0.23104175925254822,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.6953125,
|
|
"rewards/VQAORM/std": 0.46208351850509644,
|
|
"step": 1449,
|
|
"train_speed(iter/s)": 0.014475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 600.0,
|
|
"completions/mean_length": 231.0625,
|
|
"completions/min_length": 88.0,
|
|
"epoch": 0.8408234270803131,
|
|
"grad_norm": 0.5934862680628163,
|
|
"kl": 0.080810546875,
|
|
"learning_rate": 3.4283797925252407e-07,
|
|
"loss": 8.059420360950753e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3828125,
|
|
"reward_std": 0.21307221055030823,
|
|
"rewards/CSTORM/mean": 0.265625,
|
|
"rewards/CSTORM/std": 0.2504916489124298,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1450,
|
|
"train_speed(iter/s)": 0.014481
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 865.0,
|
|
"completions/mean_length": 238.8671875,
|
|
"completions/min_length": 77.0,
|
|
"epoch": 0.8414033053058858,
|
|
"grad_norm": 0.4811735479854217,
|
|
"kl": 0.0771484375,
|
|
"learning_rate": 3.4263640477282914e-07,
|
|
"loss": 7.733909296803176e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.1814119666814804,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1451,
|
|
"train_speed(iter/s)": 0.014484
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 627.0,
|
|
"completions/mean_length": 224.109375,
|
|
"completions/min_length": 55.0,
|
|
"epoch": 0.8419831835314584,
|
|
"grad_norm": 0.42775681398563614,
|
|
"kl": 0.082763671875,
|
|
"learning_rate": 3.424347705967962e-07,
|
|
"loss": 8.288821118185297e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.60546875,
|
|
"reward_std": 0.16270321607589722,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1452,
|
|
"train_speed(iter/s)": 0.014489
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 656.0,
|
|
"completions/mean_length": 247.1484375,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.842563061757031,
|
|
"grad_norm": 0.6563195837806238,
|
|
"kl": 0.06689453125,
|
|
"learning_rate": 3.4223307690238904e-07,
|
|
"loss": 6.694420153507963e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.44921875,
|
|
"reward_std": 0.2740851640701294,
|
|
"rewards/CSTORM/mean": 0.28515625,
|
|
"rewards/CSTORM/std": 0.24848829209804535,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1453,
|
|
"train_speed(iter/s)": 0.014494
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 604.0,
|
|
"completions/mean_length": 216.8359375,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.8431429399826037,
|
|
"grad_norm": 0.46273145000563154,
|
|
"kl": 0.083251953125,
|
|
"learning_rate": 3.4203132386762366e-07,
|
|
"loss": 8.316579624079168e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62109375,
|
|
"reward_std": 0.18275238573551178,
|
|
"rewards/CSTORM/mean": 0.37109375,
|
|
"rewards/CSTORM/std": 0.21957451105117798,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.75,
|
|
"rewards/VQAORM/std": 0.434714138507843,
|
|
"step": 1454,
|
|
"train_speed(iter/s)": 0.014499
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 486.0,
|
|
"completions/mean_length": 219.125,
|
|
"completions/min_length": 74.0,
|
|
"epoch": 0.8437228182081763,
|
|
"grad_norm": 0.5053308153084968,
|
|
"kl": 0.07763671875,
|
|
"learning_rate": 3.418295116705688e-07,
|
|
"loss": 7.77472960180603e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.59375,
|
|
"reward_std": 0.15193147957324982,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1455,
|
|
"train_speed(iter/s)": 0.014506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 678.0,
|
|
"completions/mean_length": 229.4453125,
|
|
"completions/min_length": 69.0,
|
|
"epoch": 0.8443026964337489,
|
|
"grad_norm": 0.5262127704583692,
|
|
"kl": 0.073486328125,
|
|
"learning_rate": 3.4162764048934505e-07,
|
|
"loss": 7.341388845816255e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.17834708094596863,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1456,
|
|
"train_speed(iter/s)": 0.014511
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 637.0,
|
|
"completions/mean_length": 235.140625,
|
|
"completions/min_length": 73.0,
|
|
"epoch": 0.8448825746593216,
|
|
"grad_norm": 0.3426390996746062,
|
|
"kl": 0.083251953125,
|
|
"learning_rate": 3.414257105021255e-07,
|
|
"loss": 8.322201028931886e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6796875,
|
|
"reward_std": 0.09439767897129059,
|
|
"rewards/CSTORM/mean": 0.3828125,
|
|
"rewards/CSTORM/std": 0.21263602375984192,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 1457,
|
|
"train_speed(iter/s)": 0.014517
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 866.0,
|
|
"completions/mean_length": 250.828125,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.8454624528848942,
|
|
"grad_norm": 0.5926979871251574,
|
|
"kl": 0.06884765625,
|
|
"learning_rate": 3.412237218871346e-07,
|
|
"loss": 6.885880429763347e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3203125,
|
|
"reward_std": 0.2282869815826416,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 1458,
|
|
"train_speed(iter/s)": 0.014522
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 528.0,
|
|
"completions/mean_length": 228.8671875,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.8460423311104668,
|
|
"grad_norm": 0.602604568788096,
|
|
"kl": 0.0771484375,
|
|
"learning_rate": 3.410216748226491e-07,
|
|
"loss": 7.722357986494899e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.62890625,
|
|
"reward_std": 0.21118974685668945,
|
|
"rewards/CSTORM/mean": 0.33984375,
|
|
"rewards/CSTORM/std": 0.23421500623226166,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7890625,
|
|
"rewards/VQAORM/std": 0.4095771610736847,
|
|
"step": 1459,
|
|
"train_speed(iter/s)": 0.014528
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 893.0,
|
|
"completions/mean_length": 216.921875,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.8466222093360394,
|
|
"grad_norm": 0.532457923706158,
|
|
"kl": 0.083251953125,
|
|
"learning_rate": 3.408195694869969e-07,
|
|
"loss": 8.316885214298964e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.35546875,
|
|
"reward_std": 0.245120570063591,
|
|
"rewards/CSTORM/mean": 0.26953125,
|
|
"rewards/CSTORM/std": 0.25021520256996155,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5859375,
|
|
"rewards/VQAORM/std": 0.49449479579925537,
|
|
"step": 1460,
|
|
"train_speed(iter/s)": 0.014533
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 644.0,
|
|
"completions/mean_length": 211.765625,
|
|
"completions/min_length": 50.0,
|
|
"epoch": 0.8472020875616121,
|
|
"grad_norm": 0.5916597450018058,
|
|
"kl": 0.0791015625,
|
|
"learning_rate": 3.406174060585576e-07,
|
|
"loss": 7.90805570431985e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.2373080551624298,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1461,
|
|
"train_speed(iter/s)": 0.014539
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 532.0,
|
|
"completions/mean_length": 211.09375,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.8477819657871847,
|
|
"grad_norm": 0.4674152654124796,
|
|
"kl": 0.0830078125,
|
|
"learning_rate": 3.404151847157619e-07,
|
|
"loss": 8.296174928545952e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.6875,
|
|
"reward_std": 0.18396098911762238,
|
|
"rewards/CSTORM/mean": 0.390625,
|
|
"rewards/CSTORM/std": 0.20751149952411652,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 1462,
|
|
"train_speed(iter/s)": 0.014545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1022.0,
|
|
"completions/mean_length": 247.6640625,
|
|
"completions/min_length": 96.0,
|
|
"epoch": 0.8483618440127573,
|
|
"grad_norm": 0.6021940550509839,
|
|
"kl": 0.078369140625,
|
|
"learning_rate": 3.4021290563709176e-07,
|
|
"loss": 7.832190749468282e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3046875,
|
|
"reward_std": 0.3155648708343506,
|
|
"rewards/CSTORM/mean": 0.2578125,
|
|
"rewards/CSTORM/std": 0.25085973739624023,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.546875,
|
|
"rewards/VQAORM/std": 0.4997538626194,
|
|
"step": 1463,
|
|
"train_speed(iter/s)": 0.01454
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 662.0,
|
|
"completions/mean_length": 202.921875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.84894172223833,
|
|
"grad_norm": 0.6727145002664586,
|
|
"kl": 0.08642578125,
|
|
"learning_rate": 3.4001056900108e-07,
|
|
"loss": 8.641450403956696e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.453125,
|
|
"reward_std": 0.2662726938724518,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1464,
|
|
"train_speed(iter/s)": 0.014546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 483.0,
|
|
"completions/mean_length": 208.453125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.8495216004639026,
|
|
"grad_norm": 0.4949309439476708,
|
|
"kl": 0.081298828125,
|
|
"learning_rate": 3.398081749863103e-07,
|
|
"loss": 8.125213935272768e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.63671875,
|
|
"reward_std": 0.17778617143630981,
|
|
"rewards/CSTORM/mean": 0.36328125,
|
|
"rewards/CSTORM/std": 0.223737433552742,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1465,
|
|
"train_speed(iter/s)": 0.014552
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 523.0,
|
|
"completions/mean_length": 200.625,
|
|
"completions/min_length": 61.0,
|
|
"epoch": 0.8501014786894752,
|
|
"grad_norm": 0.45627966922335744,
|
|
"kl": 0.081298828125,
|
|
"learning_rate": 3.3960572377141694e-07,
|
|
"loss": 8.111321221804246e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5703125,
|
|
"reward_std": 0.140625,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.71875,
|
|
"rewards/VQAORM/std": 0.4513758718967438,
|
|
"step": 1466,
|
|
"train_speed(iter/s)": 0.014559
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 555.0,
|
|
"completions/mean_length": 214.3125,
|
|
"completions/min_length": 55.0,
|
|
"epoch": 0.8506813569150479,
|
|
"grad_norm": 0.4745289460514222,
|
|
"kl": 0.078857421875,
|
|
"learning_rate": 3.3940321553508474e-07,
|
|
"loss": 7.886276580393314e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46875,
|
|
"reward_std": 0.19056488573551178,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1467,
|
|
"train_speed(iter/s)": 0.014564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1089.0,
|
|
"completions/mean_length": 246.171875,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.8512612351406205,
|
|
"grad_norm": 0.5281151314135207,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.392006504560487e-07,
|
|
"loss": 7.307112537091598e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.2637236714363098,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1468,
|
|
"train_speed(iter/s)": 0.014568
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1161.0,
|
|
"completions/mean_length": 241.171875,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.8518411133661931,
|
|
"grad_norm": 0.5447328110343184,
|
|
"kl": 0.0888671875,
|
|
"learning_rate": 3.3899802871309417e-07,
|
|
"loss": 8.863249240675941e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.7265625,
|
|
"reward_std": 0.20618988573551178,
|
|
"rewards/CSTORM/mean": 0.40625,
|
|
"rewards/CSTORM/std": 0.19592301547527313,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8203125,
|
|
"rewards/VQAORM/std": 0.3854355216026306,
|
|
"step": 1469,
|
|
"train_speed(iter/s)": 0.014572
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 967.0,
|
|
"completions/mean_length": 251.734375,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.8524209915917658,
|
|
"grad_norm": 0.5472036994194179,
|
|
"kl": 0.07958984375,
|
|
"learning_rate": 3.3879535048505644e-07,
|
|
"loss": 7.958459173096344e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.16272208094596863,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1470,
|
|
"train_speed(iter/s)": 0.014574
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 872.0,
|
|
"completions/mean_length": 233.6015625,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.8530008698173384,
|
|
"grad_norm": 0.6128481714695335,
|
|
"kl": 0.07861328125,
|
|
"learning_rate": 3.3859261595082066e-07,
|
|
"loss": 7.850873953429982e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5546875,
|
|
"reward_std": 0.3131476938724518,
|
|
"rewards/CSTORM/mean": 0.34375,
|
|
"rewards/CSTORM/std": 0.23266683518886566,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1471,
|
|
"train_speed(iter/s)": 0.01458
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 603.0,
|
|
"completions/mean_length": 218.6953125,
|
|
"completions/min_length": 90.0,
|
|
"epoch": 0.853580748042911,
|
|
"grad_norm": 0.5363356872784819,
|
|
"kl": 0.07861328125,
|
|
"learning_rate": 3.383898252893217e-07,
|
|
"loss": 7.877981988713145e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4765625,
|
|
"reward_std": 0.20795938372612,
|
|
"rewards/CSTORM/mean": 0.3046875,
|
|
"rewards/CSTORM/std": 0.2449037730693817,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1472,
|
|
"train_speed(iter/s)": 0.014586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 719.0,
|
|
"completions/mean_length": 223.3671875,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.8541606262684837,
|
|
"grad_norm": 0.5518972542806786,
|
|
"kl": 0.078125,
|
|
"learning_rate": 3.3818697867954386e-07,
|
|
"loss": 7.81815469963476e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4921875,
|
|
"reward_std": 0.22086584568023682,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1473,
|
|
"train_speed(iter/s)": 0.014592
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 597.0,
|
|
"completions/mean_length": 244.5625,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.8547405044940563,
|
|
"grad_norm": 0.6114850246534583,
|
|
"kl": 0.07177734375,
|
|
"learning_rate": 3.3798407630052097e-07,
|
|
"loss": 7.161815301515162e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.2926883101463318,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1474,
|
|
"train_speed(iter/s)": 0.014597
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 802.0,
|
|
"completions/mean_length": 240.546875,
|
|
"completions/min_length": 79.0,
|
|
"epoch": 0.8553203827196288,
|
|
"grad_norm": 0.38805703113292794,
|
|
"kl": 0.079833984375,
|
|
"learning_rate": 3.3778111833133587e-07,
|
|
"loss": 7.990885205799714e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.16712738573551178,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.625,
|
|
"rewards/VQAORM/std": 0.4860251843929291,
|
|
"step": 1475,
|
|
"train_speed(iter/s)": 0.014602
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 730.0,
|
|
"completions/mean_length": 245.8828125,
|
|
"completions/min_length": 93.0,
|
|
"epoch": 0.8559002609452016,
|
|
"grad_norm": 0.5938681703120152,
|
|
"kl": 0.074462890625,
|
|
"learning_rate": 3.375781049511208e-07,
|
|
"loss": 7.435608131345361e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.43359375,
|
|
"reward_std": 0.3191906809806824,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6328125,
|
|
"rewards/VQAORM/std": 0.4839322865009308,
|
|
"step": 1476,
|
|
"train_speed(iter/s)": 0.014607
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 741.0,
|
|
"completions/mean_length": 210.0,
|
|
"completions/min_length": 92.0,
|
|
"epoch": 0.8564801391707741,
|
|
"grad_norm": 0.5359397709546061,
|
|
"kl": 0.080810546875,
|
|
"learning_rate": 3.373750363390566e-07,
|
|
"loss": 8.077628444880247e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.65234375,
|
|
"reward_std": 0.22962738573551178,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7734375,
|
|
"rewards/VQAORM/std": 0.4202519655227661,
|
|
"step": 1477,
|
|
"train_speed(iter/s)": 0.014613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 692.0,
|
|
"completions/mean_length": 224.4765625,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.8570600173963467,
|
|
"grad_norm": 0.4478543735527658,
|
|
"kl": 0.079345703125,
|
|
"learning_rate": 3.37171912674373e-07,
|
|
"loss": 7.921776705188677e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.625,
|
|
"reward_std": 0.13872367143630981,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1478,
|
|
"train_speed(iter/s)": 0.014618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 530.0,
|
|
"completions/mean_length": 208.7421875,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.8576398956219194,
|
|
"grad_norm": 0.4137751463452922,
|
|
"kl": 0.082275390625,
|
|
"learning_rate": 3.369687341363483e-07,
|
|
"loss": 8.240071474574506e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5234375,
|
|
"reward_std": 0.1388554871082306,
|
|
"rewards/CSTORM/mean": 0.3359375,
|
|
"rewards/CSTORM/std": 0.2356877624988556,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1479,
|
|
"train_speed(iter/s)": 0.014624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 901.0,
|
|
"completions/mean_length": 224.1796875,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.858219773847492,
|
|
"grad_norm": 0.5801081124697929,
|
|
"kl": 0.073486328125,
|
|
"learning_rate": 3.3676550090430916e-07,
|
|
"loss": 7.353992259595543e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.3984375,
|
|
"reward_std": 0.28914928436279297,
|
|
"rewards/CSTORM/mean": 0.28125,
|
|
"rewards/CSTORM/std": 0.24901379644870758,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1480,
|
|
"train_speed(iter/s)": 0.014629
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 640.0,
|
|
"completions/mean_length": 216.9296875,
|
|
"completions/min_length": 78.0,
|
|
"epoch": 0.8587996520730646,
|
|
"grad_norm": 0.48023515781187626,
|
|
"kl": 0.087158203125,
|
|
"learning_rate": 3.3656221315763063e-07,
|
|
"loss": 8.716984302736819e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.17252269387245178,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1481,
|
|
"train_speed(iter/s)": 0.014635
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 528.0,
|
|
"completions/mean_length": 217.671875,
|
|
"completions/min_length": 94.0,
|
|
"epoch": 0.8593795302986373,
|
|
"grad_norm": 0.5489405414341509,
|
|
"kl": 0.072998046875,
|
|
"learning_rate": 3.363588710757358e-07,
|
|
"loss": 7.28064333088696e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.4453125,
|
|
"reward_std": 0.2703275680541992,
|
|
"rewards/CSTORM/mean": 0.2890625,
|
|
"rewards/CSTORM/std": 0.2478996366262436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.65625,
|
|
"rewards/VQAORM/std": 0.47682511806488037,
|
|
"step": 1482,
|
|
"train_speed(iter/s)": 0.014629
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 856.0,
|
|
"completions/mean_length": 223.640625,
|
|
"completions/min_length": 82.0,
|
|
"epoch": 0.8599594085242099,
|
|
"grad_norm": 0.5726649724652092,
|
|
"kl": 0.078857421875,
|
|
"learning_rate": 3.3615547483809575e-07,
|
|
"loss": 7.866759551689029e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.40625,
|
|
"reward_std": 0.2145632952451706,
|
|
"rewards/CSTORM/mean": 0.296875,
|
|
"rewards/CSTORM/std": 0.24653105437755585,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.609375,
|
|
"rewards/VQAORM/std": 0.4898075461387634,
|
|
"step": 1483,
|
|
"train_speed(iter/s)": 0.014633
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 932.0,
|
|
"completions/mean_length": 241.3984375,
|
|
"completions/min_length": 75.0,
|
|
"epoch": 0.8605392867497825,
|
|
"grad_norm": 0.5154396271546995,
|
|
"kl": 0.072265625,
|
|
"learning_rate": 3.3595202462422937e-07,
|
|
"loss": 7.213327626232058e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.10546875,
|
|
"reward_std": 0.2338140904903412,
|
|
"rewards/CSTORM/mean": 0.19921875,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.40625,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1484,
|
|
"train_speed(iter/s)": 0.014638
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 646.0,
|
|
"completions/mean_length": 232.34375,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.8611191649753551,
|
|
"grad_norm": 0.5278219185179499,
|
|
"kl": 0.08349609375,
|
|
"learning_rate": 3.3574852061370306e-07,
|
|
"loss": 8.362461085198447e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.578125,
|
|
"reward_std": 0.22032380104064941,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7265625,
|
|
"rewards/VQAORM/std": 0.447474867105484,
|
|
"step": 1485,
|
|
"train_speed(iter/s)": 0.014644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 553.0,
|
|
"completions/mean_length": 217.078125,
|
|
"completions/min_length": 100.0,
|
|
"epoch": 0.8616990432009278,
|
|
"grad_norm": 0.5211203316576062,
|
|
"kl": 0.077880859375,
|
|
"learning_rate": 3.3554496298613095e-07,
|
|
"loss": 7.78373796492815e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.46484375,
|
|
"reward_std": 0.2372892051935196,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1486,
|
|
"train_speed(iter/s)": 0.014649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1602.0,
|
|
"completions/mean_length": 242.1640625,
|
|
"completions/min_length": 58.0,
|
|
"epoch": 0.8622789214265004,
|
|
"grad_norm": 0.6237470281997237,
|
|
"kl": 0.080810546875,
|
|
"learning_rate": 3.353413519211742e-07,
|
|
"loss": 8.083951252046973e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.484375,
|
|
"reward_std": 0.22750747203826904,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1487,
|
|
"train_speed(iter/s)": 0.014651
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 862.0,
|
|
"completions/mean_length": 231.765625,
|
|
"completions/min_length": 76.0,
|
|
"epoch": 0.862858799652073,
|
|
"grad_norm": 0.5812196399201258,
|
|
"kl": 0.19970703125,
|
|
"learning_rate": 3.351376875985414e-07,
|
|
"loss": 0.0001991725730476901,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.20498128235340118,
|
|
"rewards/CSTORM/mean": 0.3125,
|
|
"rewards/CSTORM/std": 0.24301259219646454,
|
|
"rewards/FMTORM/mean": 0.49609375,
|
|
"rewards/FMTORM/std": 0.04419417306780815,
|
|
"rewards/VQAORM/mean": 0.671875,
|
|
"rewards/VQAORM/std": 0.4713755249977112,
|
|
"step": 1488,
|
|
"train_speed(iter/s)": 0.014606
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1119.0,
|
|
"completions/mean_length": 237.53125,
|
|
"completions/min_length": 85.0,
|
|
"epoch": 0.8634386778776457,
|
|
"grad_norm": 0.5450179970551204,
|
|
"kl": 0.075439453125,
|
|
"learning_rate": 3.3493397019798797e-07,
|
|
"loss": 7.526886474806815e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.58984375,
|
|
"reward_std": 0.21883676946163177,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.734375,
|
|
"rewards/VQAORM/std": 0.44340085983276367,
|
|
"step": 1489,
|
|
"train_speed(iter/s)": 0.01461
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1274.0,
|
|
"completions/mean_length": 248.4375,
|
|
"completions/min_length": 84.0,
|
|
"epoch": 0.8640185561032183,
|
|
"grad_norm": 0.40668038944300955,
|
|
"kl": 0.078857421875,
|
|
"learning_rate": 3.3473019989931623e-07,
|
|
"loss": 7.877044845372438e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.41796875,
|
|
"reward_std": 0.1556890904903412,
|
|
"rewards/CSTORM/mean": 0.30078125,
|
|
"rewards/CSTORM/std": 0.2457500398159027,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6171875,
|
|
"rewards/VQAORM/std": 0.4879830479621887,
|
|
"step": 1490,
|
|
"train_speed(iter/s)": 0.014614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 606.0,
|
|
"completions/mean_length": 247.2109375,
|
|
"completions/min_length": 99.0,
|
|
"epoch": 0.8645984343287909,
|
|
"grad_norm": 0.4609477202040844,
|
|
"kl": 0.07275390625,
|
|
"learning_rate": 3.345263768823753e-07,
|
|
"loss": 7.277615077327937e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.45703125,
|
|
"reward_std": 0.1869390904903412,
|
|
"rewards/CSTORM/mean": 0.30859375,
|
|
"rewards/CSTORM/std": 0.24399152398109436,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6484375,
|
|
"rewards/VQAORM/std": 0.4793342351913452,
|
|
"step": 1491,
|
|
"train_speed(iter/s)": 0.014619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 523.0,
|
|
"completions/mean_length": 241.3046875,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.8651783125543636,
|
|
"grad_norm": 0.5498882481217002,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 3.343225013270605e-07,
|
|
"loss": 7.052485307212919e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.67578125,
|
|
"reward_std": 0.2204744815826416,
|
|
"rewards/CSTORM/mean": 0.37890625,
|
|
"rewards/CSTORM/std": 0.2150452584028244,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.796875,
|
|
"rewards/VQAORM/std": 0.40390563011169434,
|
|
"step": 1492,
|
|
"train_speed(iter/s)": 0.014614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 799.0,
|
|
"completions/mean_length": 239.7890625,
|
|
"completions/min_length": 83.0,
|
|
"epoch": 0.8657581907799362,
|
|
"grad_norm": 0.5658463144885746,
|
|
"kl": 0.070556640625,
|
|
"learning_rate": 3.3411857341331395e-07,
|
|
"loss": 7.070018182275817e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.48046875,
|
|
"reward_std": 0.24069641530513763,
|
|
"rewards/CSTORM/mean": 0.31640625,
|
|
"rewards/CSTORM/std": 0.2419661432504654,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6640625,
|
|
"rewards/VQAORM/std": 0.47417303919792175,
|
|
"step": 1493,
|
|
"train_speed(iter/s)": 0.014619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 476.0,
|
|
"completions/mean_length": 199.4375,
|
|
"completions/min_length": 72.0,
|
|
"epoch": 0.8663380690055088,
|
|
"grad_norm": 0.52252750422572,
|
|
"kl": 0.0888671875,
|
|
"learning_rate": 3.3391459332112373e-07,
|
|
"loss": 8.901901310309768e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.671875,
|
|
"reward_std": 0.19233438372612,
|
|
"rewards/CSTORM/mean": 0.3671875,
|
|
"rewards/CSTORM/std": 0.22170042991638184,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.8046875,
|
|
"rewards/VQAORM/std": 0.3979988098144531,
|
|
"step": 1494,
|
|
"train_speed(iter/s)": 0.014625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 895.0,
|
|
"completions/mean_length": 245.25,
|
|
"completions/min_length": 80.0,
|
|
"epoch": 0.8669179472310815,
|
|
"grad_norm": 0.5949391271881517,
|
|
"kl": 0.06494140625,
|
|
"learning_rate": 3.3371056123052395e-07,
|
|
"loss": 6.496525747934356e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5078125,
|
|
"reward_std": 0.2530648708343506,
|
|
"rewards/CSTORM/mean": 0.3203125,
|
|
"rewards/CSTORM/std": 0.2408512979745865,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.6875,
|
|
"rewards/VQAORM/std": 0.4653336703777313,
|
|
"step": 1495,
|
|
"train_speed(iter/s)": 0.01463
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 869.0,
|
|
"completions/mean_length": 225.2578125,
|
|
"completions/min_length": 66.0,
|
|
"epoch": 0.8674978254566541,
|
|
"grad_norm": 0.355938695785829,
|
|
"kl": 0.078857421875,
|
|
"learning_rate": 3.3350647732159465e-07,
|
|
"loss": 7.883230864536017e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.32421875,
|
|
"reward_std": 0.08325667679309845,
|
|
"rewards/CSTORM/mean": 0.26171875,
|
|
"rewards/CSTORM/std": 0.25070643424987793,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.5625,
|
|
"rewards/VQAORM/std": 0.49802759289741516,
|
|
"step": 1496,
|
|
"train_speed(iter/s)": 0.014634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 865.0,
|
|
"completions/mean_length": 215.4375,
|
|
"completions/min_length": 65.0,
|
|
"epoch": 0.8680777036822267,
|
|
"grad_norm": 0.635674056932185,
|
|
"kl": 0.08251953125,
|
|
"learning_rate": 3.333023417744618e-07,
|
|
"loss": 8.244451601058245e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.61328125,
|
|
"reward_std": 0.24015437066555023,
|
|
"rewards/CSTORM/mean": 0.35546875,
|
|
"rewards/CSTORM/std": 0.22755412757396698,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1497,
|
|
"train_speed(iter/s)": 0.014639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 597.0,
|
|
"completions/mean_length": 221.2578125,
|
|
"completions/min_length": 87.0,
|
|
"epoch": 0.8686575819077994,
|
|
"grad_norm": 0.49430627331229443,
|
|
"kl": 0.07861328125,
|
|
"learning_rate": 3.330981547692966e-07,
|
|
"loss": 7.857532182242721e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.609375,
|
|
"reward_std": 0.18736818432807922,
|
|
"rewards/CSTORM/mean": 0.3515625,
|
|
"rewards/CSTORM/std": 0.22933810949325562,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7578125,
|
|
"rewards/VQAORM/std": 0.4300905168056488,
|
|
"step": 1498,
|
|
"train_speed(iter/s)": 0.014645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 671.0,
|
|
"completions/mean_length": 232.640625,
|
|
"completions/min_length": 81.0,
|
|
"epoch": 0.869237460133372,
|
|
"grad_norm": 0.5965171821041625,
|
|
"kl": 0.08447265625,
|
|
"learning_rate": 3.32893916486316e-07,
|
|
"loss": 8.449883171124384e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.5390625,
|
|
"reward_std": 0.24445399641990662,
|
|
"rewards/CSTORM/mean": 0.328125,
|
|
"rewards/CSTORM/std": 0.23841255903244019,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.7109375,
|
|
"rewards/VQAORM/std": 0.45510825514793396,
|
|
"step": 1499,
|
|
"train_speed(iter/s)": 0.014651
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1284.0,
|
|
"completions/mean_length": 245.6640625,
|
|
"completions/min_length": 91.0,
|
|
"epoch": 0.8698173383589446,
|
|
"grad_norm": 0.5631007853217043,
|
|
"kl": 0.076416015625,
|
|
"learning_rate": 3.3268962710578194e-07,
|
|
"loss": 7.648733298992738e-05,
|
|
"memory(GiB)": 52.62,
|
|
"reward": 1.38671875,
|
|
"reward_std": 0.3191906809806824,
|
|
"rewards/CSTORM/mean": 0.29296875,
|
|
"rewards/CSTORM/std": 0.24724739789962769,
|
|
"rewards/FMTORM/mean": 0.5,
|
|
"rewards/FMTORM/std": 0.0,
|
|
"rewards/VQAORM/mean": 0.59375,
|
|
"rewards/VQAORM/std": 0.4930621087551117,
|
|
"step": 1500,
|
|
"train_speed(iter/s)": 0.014654
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 3448,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 2,
|
|
"save_steps": 100,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|