f07657fce7
Model: Asib1/Qwen2.5-0.5B-Instruct-Gensyn-Swarm-pensive_leggy_ant Source: Original Platform
824 lines
34 KiB
JSON
824 lines
34 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.43956043956043955,
|
|
"eval_steps": 500,
|
|
"global_step": 20,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 43.0,
|
|
"completions/max_terminated_length": 43.0,
|
|
"completions/mean_length": 27.25,
|
|
"completions/mean_terminated_length": 27.25,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.02197802197802198,
|
|
"grad_norm": 22.348234176635742,
|
|
"kl": 0.0,
|
|
"learning_rate": 0.0,
|
|
"loss": 0.1788,
|
|
"num_tokens": 621.0,
|
|
"reward": 0.1662130355834961,
|
|
"reward_std": 0.008482580073177814,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.040213026106357574,
|
|
"rewards/question_recreation_reward_func/std": 0.011011077091097832,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 104.0,
|
|
"completions/max_terminated_length": 104.0,
|
|
"completions/mean_length": 43.0,
|
|
"completions/mean_terminated_length": 43.0,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"epoch": 0.04395604395604396,
|
|
"grad_norm": 22.66720962524414,
|
|
"kl": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08,
|
|
"num_tokens": 1305.0,
|
|
"reward": 0.3151491582393646,
|
|
"reward_std": 0.06959080696105957,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.1891491413116455,
|
|
"rewards/question_recreation_reward_func/std": 0.09147033840417862,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 47.0,
|
|
"completions/max_terminated_length": 47.0,
|
|
"completions/mean_length": 24.75,
|
|
"completions/mean_terminated_length": 24.75,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"epoch": 0.06593406593406594,
|
|
"grad_norm": 27.277257919311523,
|
|
"kl": 0.014962893881602213,
|
|
"learning_rate": 9.931806517013612e-07,
|
|
"loss": -0.2371,
|
|
"num_tokens": 1916.0,
|
|
"reward": 0.15970052778720856,
|
|
"reward_std": 0.00753402803093195,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.033700525760650635,
|
|
"rewards/question_recreation_reward_func/std": 0.012940811924636364,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 79.0,
|
|
"completions/max_terminated_length": 79.0,
|
|
"completions/mean_length": 36.0,
|
|
"completions/mean_terminated_length": 36.0,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"epoch": 0.08791208791208792,
|
|
"grad_norm": 40.29623794555664,
|
|
"kl": 0.004107625340111554,
|
|
"learning_rate": 9.729086208503173e-07,
|
|
"loss": -0.3742,
|
|
"num_tokens": 2572.0,
|
|
"reward": 0.1578778773546219,
|
|
"reward_std": 0.023526929318904877,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.03187786787748337,
|
|
"rewards/question_recreation_reward_func/std": 0.027124471962451935,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 125.0,
|
|
"completions/max_terminated_length": 125.0,
|
|
"completions/mean_length": 45.25,
|
|
"completions/mean_terminated_length": 45.25,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"epoch": 0.10989010989010989,
|
|
"grad_norm": 18.357593536376953,
|
|
"kl": 0.004576658713631332,
|
|
"learning_rate": 9.397368756032444e-07,
|
|
"loss": 0.2573,
|
|
"num_tokens": 3265.0,
|
|
"reward": 0.17751163244247437,
|
|
"reward_std": 0.009721913374960423,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.05151163041591644,
|
|
"rewards/question_recreation_reward_func/std": 0.028811020776629448,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 98.0,
|
|
"completions/max_terminated_length": 98.0,
|
|
"completions/mean_length": 43.75,
|
|
"completions/mean_terminated_length": 43.75,
|
|
"completions/min_length": 17.0,
|
|
"completions/min_terminated_length": 17.0,
|
|
"epoch": 0.13186813186813187,
|
|
"grad_norm": 20.5982723236084,
|
|
"kl": 0.014701616251841187,
|
|
"learning_rate": 8.945702546981968e-07,
|
|
"loss": 0.148,
|
|
"num_tokens": 3952.0,
|
|
"reward": 0.16695590317249298,
|
|
"reward_std": 0.003352080937474966,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.04095589742064476,
|
|
"rewards/question_recreation_reward_func/std": 0.015096832998096943,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 61.0,
|
|
"completions/max_terminated_length": 61.0,
|
|
"completions/mean_length": 48.5,
|
|
"completions/mean_terminated_length": 48.5,
|
|
"completions/min_length": 41.0,
|
|
"completions/min_terminated_length": 41.0,
|
|
"epoch": 0.15384615384615385,
|
|
"grad_norm": 14.144696235656738,
|
|
"kl": 0.003702761954627931,
|
|
"learning_rate": 8.386407858128706e-07,
|
|
"loss": 0.0654,
|
|
"num_tokens": 4658.0,
|
|
"reward": 0.26975181698799133,
|
|
"reward_std": 0.0007923027733340859,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.1437518149614334,
|
|
"rewards/question_recreation_reward_func/std": 0.1348366141319275,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 188.0,
|
|
"completions/max_terminated_length": 188.0,
|
|
"completions/mean_length": 83.25,
|
|
"completions/mean_terminated_length": 83.25,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"epoch": 0.17582417582417584,
|
|
"grad_norm": 26.052366256713867,
|
|
"kl": 0.041728515177965164,
|
|
"learning_rate": 7.734740790612136e-07,
|
|
"loss": 0.1201,
|
|
"num_tokens": 5503.0,
|
|
"reward": 0.15764988958835602,
|
|
"reward_std": 0.006401078775525093,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.031649891287088394,
|
|
"rewards/question_recreation_reward_func/std": 0.008277686312794685,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 50.0,
|
|
"completions/max_terminated_length": 50.0,
|
|
"completions/mean_length": 25.5,
|
|
"completions/mean_terminated_length": 25.5,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"epoch": 0.1978021978021978,
|
|
"grad_norm": 41.049705505371094,
|
|
"kl": 0.048197535099461675,
|
|
"learning_rate": 7.008477123264847e-07,
|
|
"loss": -0.3183,
|
|
"num_tokens": 6117.0,
|
|
"reward": 0.1554895043373108,
|
|
"reward_std": 0.005939033348113298,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.029489504173398018,
|
|
"rewards/question_recreation_reward_func/std": 0.0072249844670295715,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 184.0,
|
|
"completions/max_terminated_length": 184.0,
|
|
"completions/mean_length": 115.75,
|
|
"completions/mean_terminated_length": 115.75,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"epoch": 0.21978021978021978,
|
|
"grad_norm": 12.243282318115234,
|
|
"kl": 0.010378418024629354,
|
|
"learning_rate": 6.227427435703995e-07,
|
|
"loss": 0.3581,
|
|
"num_tokens": 7092.0,
|
|
"reward": 0.3181113600730896,
|
|
"reward_std": 0.0568658784031868,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.19211135804653168,
|
|
"rewards/question_recreation_reward_func/std": 0.06996887922286987,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 29.0,
|
|
"completions/max_terminated_length": 29.0,
|
|
"completions/mean_length": 19.75,
|
|
"completions/mean_terminated_length": 19.75,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"epoch": 0.24175824175824176,
|
|
"grad_norm": 31.97664451599121,
|
|
"kl": 0.03674608655273914,
|
|
"learning_rate": 5.412896727361662e-07,
|
|
"loss": -0.2198,
|
|
"num_tokens": 7683.0,
|
|
"reward": 0.2507641613483429,
|
|
"reward_std": 0.09787573665380478,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.12476415932178497,
|
|
"rewards/question_recreation_reward_func/std": 0.0912269577383995,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 84.0,
|
|
"completions/max_terminated_length": 84.0,
|
|
"completions/mean_length": 48.0,
|
|
"completions/mean_terminated_length": 48.0,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"epoch": 0.26373626373626374,
|
|
"grad_norm": 17.758264541625977,
|
|
"kl": 0.02334656985476613,
|
|
"learning_rate": 4.5871032726383385e-07,
|
|
"loss": -0.0316,
|
|
"num_tokens": 8387.0,
|
|
"reward": 0.18516191840171814,
|
|
"reward_std": 0.0012881024740636349,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.059161920100450516,
|
|
"rewards/question_recreation_reward_func/std": 0.0013300965074449778,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 30.0,
|
|
"completions/max_terminated_length": 30.0,
|
|
"completions/mean_length": 23.0,
|
|
"completions/mean_terminated_length": 23.0,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"epoch": 0.2857142857142857,
|
|
"grad_norm": 24.956192016601562,
|
|
"kl": 0.022823655512183905,
|
|
"learning_rate": 3.772572564296004e-07,
|
|
"loss": -0.0155,
|
|
"num_tokens": 8991.0,
|
|
"reward": 0.15829095244407654,
|
|
"reward_std": 0.004669444169849157,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.03229094669222832,
|
|
"rewards/question_recreation_reward_func/std": 0.005796445999294519,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 35.0,
|
|
"completions/max_terminated_length": 35.0,
|
|
"completions/mean_length": 25.5,
|
|
"completions/mean_terminated_length": 25.5,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"epoch": 0.3076923076923077,
|
|
"grad_norm": 31.84866714477539,
|
|
"kl": 0.011406762059777975,
|
|
"learning_rate": 2.9915228767351535e-07,
|
|
"loss": 0.1413,
|
|
"num_tokens": 9605.0,
|
|
"reward": 0.14927050471305847,
|
|
"reward_std": 0.013867171481251717,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.023270506411790848,
|
|
"rewards/question_recreation_reward_func/std": 0.0208793506026268,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 66.0,
|
|
"completions/max_terminated_length": 66.0,
|
|
"completions/mean_length": 28.75,
|
|
"completions/mean_terminated_length": 28.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.32967032967032966,
|
|
"grad_norm": 19.86272430419922,
|
|
"kl": 0.013966037891805172,
|
|
"learning_rate": 2.2652592093878665e-07,
|
|
"loss": 0.2861,
|
|
"num_tokens": 10232.0,
|
|
"reward": 0.5240224003791809,
|
|
"reward_std": 0.011384031735360622,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.3980224132537842,
|
|
"rewards/question_recreation_reward_func/std": 0.15406456589698792,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 67.0,
|
|
"completions/max_terminated_length": 67.0,
|
|
"completions/mean_length": 36.75,
|
|
"completions/mean_terminated_length": 36.75,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"epoch": 0.3516483516483517,
|
|
"grad_norm": 16.492538452148438,
|
|
"kl": 0.0071347299963235855,
|
|
"learning_rate": 1.6135921418712955e-07,
|
|
"loss": -0.1412,
|
|
"num_tokens": 10891.0,
|
|
"reward": 0.14584042131900787,
|
|
"reward_std": 0.0016124435933306813,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.019840413704514503,
|
|
"rewards/question_recreation_reward_func/std": 0.007639859337359667,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 59.0,
|
|
"completions/max_terminated_length": 59.0,
|
|
"completions/mean_length": 34.5,
|
|
"completions/mean_terminated_length": 34.5,
|
|
"completions/min_length": 21.0,
|
|
"completions/min_terminated_length": 21.0,
|
|
"epoch": 0.37362637362637363,
|
|
"grad_norm": 26.692140579223633,
|
|
"kl": 0.012246189173310995,
|
|
"learning_rate": 1.0542974530180327e-07,
|
|
"loss": -0.0637,
|
|
"num_tokens": 11541.0,
|
|
"reward": 0.14543770253658295,
|
|
"reward_std": 0.006278357934206724,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.019437694922089577,
|
|
"rewards/question_recreation_reward_func/std": 0.005619715433567762,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 38.0,
|
|
"completions/max_terminated_length": 38.0,
|
|
"completions/mean_length": 28.5,
|
|
"completions/mean_terminated_length": 28.5,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"epoch": 0.3956043956043956,
|
|
"grad_norm": 32.2912483215332,
|
|
"kl": 0.02024045307189226,
|
|
"learning_rate": 6.026312439675551e-08,
|
|
"loss": 0.2078,
|
|
"num_tokens": 12167.0,
|
|
"reward": 0.30906468629837036,
|
|
"reward_std": 0.029750796034932137,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.18306468427181244,
|
|
"rewards/question_recreation_reward_func/std": 0.07680102437734604,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 74.0,
|
|
"completions/max_terminated_length": 74.0,
|
|
"completions/mean_length": 31.25,
|
|
"completions/mean_terminated_length": 31.25,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"epoch": 0.4175824175824176,
|
|
"grad_norm": 29.64141273498535,
|
|
"kl": 0.013999502174556255,
|
|
"learning_rate": 2.7091379149682682e-08,
|
|
"loss": -0.2169,
|
|
"num_tokens": 12804.0,
|
|
"reward": 0.29109445214271545,
|
|
"reward_std": 0.0344494953751564,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.16509445011615753,
|
|
"rewards/question_recreation_reward_func/std": 0.05098514258861542,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 65.0,
|
|
"completions/max_terminated_length": 65.0,
|
|
"completions/mean_length": 34.25,
|
|
"completions/mean_terminated_length": 34.25,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"epoch": 0.43956043956043955,
|
|
"grad_norm": 22.295286178588867,
|
|
"kl": 0.03087549190968275,
|
|
"learning_rate": 6.819348298638839e-09,
|
|
"loss": -0.142,
|
|
"num_tokens": 13453.0,
|
|
"reward": 0.16607344150543213,
|
|
"reward_std": 0.008879171684384346,
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
|
"rewards/consensus_reward_func/std": 0.0,
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
|
"rewards/question_recreation_reward_func/mean": 0.0400734469294548,
|
|
"rewards/question_recreation_reward_func/std": 0.014095885679125786,
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
|
"rewards/xmlcount_reward_func/mean": 0.12600000202655792,
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.43956043956043955,
|
|
"step": 20,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.004136198572814465,
|
|
"train_runtime": 6309.5264,
|
|
"train_samples_per_second": 0.013,
|
|
"train_steps_per_second": 0.003
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 20,
|
|
"num_input_tokens_seen": 13453,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 25,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|