17368 lines
807 KiB
JSON
17368 lines
807 KiB
JSON
{
|
|
"best_metric": 0.6866666674613953,
|
|
"best_model_checkpoint": "/scratch-ssd/lucelo/max-ent-llms/data/Qwen-2.5-7B-RL-LACPO-NoBaselineAlpha1/checkpoint-19",
|
|
"epoch": 0.9873417721518988,
|
|
"eval_steps": 1,
|
|
"global_step": 39,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"accuracy_reward": 0.640625,
|
|
"accuracy_reward/correct": 0.9999999403953552,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.23142997920513153,
|
|
"adam_stats/lr_effective_max": 7.905724487500265e-06,
|
|
"adam_stats/lr_effective_mean": -3.657434077819488e-11,
|
|
"adam_stats/lr_effective_min": -7.905713573563844e-06,
|
|
"adam_stats/m_t_max": 0.011035159230232239,
|
|
"adam_stats/m_t_mean": -7.945365199102383e-11,
|
|
"adam_stats/m_t_min": -0.007421876769512892,
|
|
"adam_stats/v_t_max": 1.2177310964034405e-05,
|
|
"adam_stats/v_t_mean": 2.893036287962558e-13,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.640625,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.23142997920513153,
|
|
"all_logprobs": -0.16401661932468414,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": -2.276897430419922e-05,
|
|
"all_logprobs/min": -10.125,
|
|
"all_logprobs/p25": -0.0179443359375,
|
|
"all_logprobs/p75": -4.76837158203125e-07,
|
|
"all_logprobs/var": 0.2920554578304291,
|
|
"answer_logprob_reward": 2.7831039428710938,
|
|
"answer_logprob_reward/correct": 3.034297466278076,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.10153310000896454,
|
|
"answer_logprob_reward/incorrect": 2.335324287414551,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/min": 1.2321436405181885,
|
|
"answer_logprob_reward/incorrect/p25": 1.3862943649291992,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.7438019514083862,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.2321436405181885,
|
|
"answer_logprob_reward/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.44273096323013306,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 583.7760620117188,
|
|
"completion_length/correct": 521.78857421875,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 452.0,
|
|
"completion_length/correct/min": 12.0,
|
|
"completion_length/correct/p25": 338.5,
|
|
"completion_length/correct/p75": 686.0,
|
|
"completion_length/correct/var": 68040.7734375,
|
|
"completion_length/incorrect": 694.275390625,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 808.0,
|
|
"completion_length/incorrect/min": 24.0,
|
|
"completion_length/incorrect/p25": 340.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 129540.015625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 525.0,
|
|
"completion_length/min": 12.0,
|
|
"completion_length/p25": 339.25,
|
|
"completion_length/p75": 873.5,
|
|
"completion_length/var": 96464.9765625,
|
|
"epoch": 0.02531645569620253,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 112085.0,
|
|
"grad_norm": 0.7914038300514221,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.5e-06,
|
|
"len_reward": 0.01521260291337967,
|
|
"len_reward/correct": 0.11566465348005295,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.14369310438632965,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.023282986134290695,
|
|
"len_reward/correct/p75": 0.2884446978569031,
|
|
"len_reward/correct/var": 0.05871717259287834,
|
|
"len_reward/incorrect": -0.16385412216186523,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.12603966891765594,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.29398593306541443,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.029935725033283234,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.0,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.16415677964687347,
|
|
"len_reward/p75": 0.21801316738128662,
|
|
"len_reward/var": 0.06624475121498108,
|
|
"loss": -0.6406,
|
|
"mean_logprobs": -0.197265625,
|
|
"mean_logprobs/var": 0.0306396484375,
|
|
"missing_response_penalty": -0.1822916716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.5072463750839233,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.25362321734428406,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.14984185993671417,
|
|
"non_smoothed_logprobs/min": -16.75,
|
|
"num_completions/total": 192,
|
|
"policy_loss": -0.640625,
|
|
"policy_loss/max": 0.0,
|
|
"policy_loss/median": -1.0,
|
|
"policy_loss/min": -1.0,
|
|
"policy_loss/p25": -1.0,
|
|
"policy_loss/p75": 0.0,
|
|
"policy_loss/var": 0.23142997920513153,
|
|
"pred_value_log_probs": -0.6782176494598389,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.7578125,
|
|
"pred_value_log_probs/min": -1.4609375,
|
|
"pred_value_log_probs/p25": -0.7578125,
|
|
"pred_value_log_probs/p75": -0.7578125,
|
|
"pred_value_log_probs/var": 0.046743351966142654,
|
|
"reward": 0.640625,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.23142997920513153,
|
|
"rewards/accuracy_reward": 0.640625,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.23142997920513153,
|
|
"rewards/answer_logprob_reward": 2.7831039428710938,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.2321436405181885,
|
|
"rewards/answer_logprob_reward/p25": 2.4849066734313965,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.44273096323013306,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.01521260291337967,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.0,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.16415677964687347,
|
|
"rewards/len_reward/p75": 0.21801316738128662,
|
|
"rewards/len_reward/var": 0.06624475121498108,
|
|
"rewards/missing_response_penalty": -0.1822916716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.14984185993671417,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 1,
|
|
"value_loss": 2.7890625,
|
|
"values": -0.7917071580886841,
|
|
"values/max": 0.73681640625,
|
|
"values/median": -0.6884765625,
|
|
"values/min": -2.0000112056732178,
|
|
"values/p25": -0.6884765625,
|
|
"values/p75": -0.6884765625,
|
|
"values/var": 0.12838509678840637
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 7.905724487500265e-06,
|
|
"adam_stats/lr_effective_mean": -3.657434077819488e-11,
|
|
"adam_stats/lr_effective_min": -7.905713573563844e-06,
|
|
"adam_stats/m_t_max": 0.011035159230232239,
|
|
"adam_stats/m_t_mean": -7.945365199102383e-11,
|
|
"adam_stats/m_t_min": -0.007421876769512892,
|
|
"adam_stats/v_t_max": 1.2177310964034405e-05,
|
|
"adam_stats/v_t_mean": 2.893036287962558e-13,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.02531645569620253,
|
|
"eval_accuracy_reward": 0.4399999976158142,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 0.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.24722407758235931,
|
|
"eval_advantages": 0.4399999976158142,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 0.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.24722407758235931,
|
|
"eval_all_logprobs": -0.17103739082813263,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": -4.601478576660156e-05,
|
|
"eval_all_logprobs/min": -12.125,
|
|
"eval_all_logprobs/p25": -0.0211181640625,
|
|
"eval_all_logprobs/p75": -7.152557373046875e-07,
|
|
"eval_all_logprobs/var": 0.31482362747192383,
|
|
"eval_answer_logprob_reward": 2.664313316345215,
|
|
"eval_answer_logprob_reward/correct": 3.161113739013672,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.03698143735527992,
|
|
"eval_answer_logprob_reward/incorrect": 2.273969888687134,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/min": 0.7339691519737244,
|
|
"eval_answer_logprob_reward/incorrect/p25": 1.272965669631958,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.9783580303192139,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 0.7339691519737244,
|
|
"eval_answer_logprob_reward/p25": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.7572145462036133,
|
|
"eval_completion_length": 576.9166870117188,
|
|
"eval_completion_length/correct": 561.4166870117188,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 522.0,
|
|
"eval_completion_length/correct/min": 66.0,
|
|
"eval_completion_length/correct/p25": 344.0,
|
|
"eval_completion_length/correct/p75": 720.0,
|
|
"eval_completion_length/correct/var": 73380.4921875,
|
|
"eval_completion_length/incorrect": 589.0952758789062,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 556.0,
|
|
"eval_completion_length/incorrect/min": 8.0,
|
|
"eval_completion_length/incorrect/p25": 212.25,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 139438.125,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 531.0,
|
|
"eval_completion_length/min": 8.0,
|
|
"eval_completion_length/p25": 301.75,
|
|
"eval_completion_length/p75": 974.5,
|
|
"eval_completion_length/var": 110219.5390625,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 173075.0,
|
|
"eval_len_reward": -0.042816244065761566,
|
|
"eval_len_reward/correct": 0.06277946382761002,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.09141184389591217,
|
|
"eval_len_reward/correct/min": -0.4624646008014679,
|
|
"eval_len_reward/correct/p25": -0.08297038823366165,
|
|
"eval_len_reward/correct/p75": 0.23608985543251038,
|
|
"eval_len_reward/correct/var": 0.054971843957901,
|
|
"eval_len_reward/incorrect": -0.12578429281711578,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": 0.0,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.24150700867176056,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03059995174407959,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.18369412422180176,
|
|
"eval_len_reward/p75": 0.03296111524105072,
|
|
"eval_len_reward/var": 0.04996596649289131,
|
|
"eval_mean_logprobs": -0.22265625,
|
|
"eval_mean_logprobs/var": 0.0419921875,
|
|
"eval_missing_response_penalty": -0.26333335041999817,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.4702380895614624,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.25060591101646423,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": -1.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.19463768601417542,
|
|
"eval_num_completions/total": 300,
|
|
"eval_pred_value_log_probs": -0.7142305970191956,
|
|
"eval_pred_value_log_probs/max": -2.2649765014648438e-06,
|
|
"eval_pred_value_log_probs/median": -0.7578125,
|
|
"eval_pred_value_log_probs/min": -1.234375,
|
|
"eval_pred_value_log_probs/p25": -0.7578125,
|
|
"eval_pred_value_log_probs/p75": -0.7578125,
|
|
"eval_pred_value_log_probs/var": 0.022694004699587822,
|
|
"eval_reward": 0.4399999976158142,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 0.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.24722407758235931,
|
|
"eval_rewards/accuracy_reward": 0.4399999976158142,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 0.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.24722407758235931,
|
|
"eval_rewards/answer_logprob_reward": 2.664313316345215,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 0.7339691519737244,
|
|
"eval_rewards/answer_logprob_reward/p25": 1.8325815200805664,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.7572145462036133,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": -0.042816244065761566,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.18369412422180176,
|
|
"eval_rewards/len_reward/p75": 0.03296111524105072,
|
|
"eval_rewards/len_reward/var": 0.04996596649289131,
|
|
"eval_rewards/missing_response_penalty": -0.26333335041999817,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": -1.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.19463768601417542,
|
|
"eval_runtime": 171.0734,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.7239292860031128,
|
|
"eval_values/max": 1.9339599609375,
|
|
"eval_values/median": -0.6884765625,
|
|
"eval_values/min": -1.9770984649658203,
|
|
"eval_values/p25": -0.6884765625,
|
|
"eval_values/p75": -0.6884765625,
|
|
"eval_values/var": 0.09213504940271378,
|
|
"step": 1
|
|
},
|
|
{
|
|
"accuracy_reward": 0.6197916865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.2368837296962738,
|
|
"adam_stats/lr_effective_max": 2.1276326151564717e-05,
|
|
"adam_stats/lr_effective_mean": 7.12119209086981e-12,
|
|
"adam_stats/lr_effective_min": -2.1276304323691875e-05,
|
|
"adam_stats/m_t_max": 0.0102612329646945,
|
|
"adam_stats/m_t_mean": -4.9798998258410165e-11,
|
|
"adam_stats/m_t_min": -0.007576906122267246,
|
|
"adam_stats/v_t_max": 1.2175996744190343e-05,
|
|
"adam_stats/v_t_mean": 3.826528937461715e-13,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.6197916865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.2368837296962738,
|
|
"all_logprobs": -0.15524685382843018,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": -2.2172927856445312e-05,
|
|
"all_logprobs/min": -9.25,
|
|
"all_logprobs/p25": -0.0135498046875,
|
|
"all_logprobs/p75": -4.76837158203125e-07,
|
|
"all_logprobs/var": 0.2705397605895996,
|
|
"answer_logprob_reward": 2.6923842430114746,
|
|
"answer_logprob_reward/correct": 3.0770535469055176,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.07541394978761673,
|
|
"answer_logprob_reward/incorrect": 2.0653202533721924,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/min": 1.0986123085021973,
|
|
"answer_logprob_reward/incorrect/p25": 1.0986123085021973,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.8078325986862183,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.0986123085021973,
|
|
"answer_logprob_reward/p25": 2.079441547393799,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.593589186668396,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 641.9896240234375,
|
|
"completion_length/correct": 566.2269287109375,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 513.0,
|
|
"completion_length/correct/min": 117.0,
|
|
"completion_length/correct/p25": 372.5,
|
|
"completion_length/correct/p75": 710.5,
|
|
"completion_length/correct/var": 61870.33203125,
|
|
"completion_length/incorrect": 765.4931640625,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 17.0,
|
|
"completion_length/incorrect/p25": 471.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 116871.15625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 570.0,
|
|
"completion_length/min": 17.0,
|
|
"completion_length/p25": 390.25,
|
|
"completion_length/p75": 1024.0,
|
|
"completion_length/var": 91685.640625,
|
|
"epoch": 0.05063291139240506,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 235347.0,
|
|
"grad_norm": 0.48119205236434937,
|
|
"kl": 0.0,
|
|
"learning_rate": 5e-06,
|
|
"len_reward": -0.0022739358246326447,
|
|
"len_reward/correct": 0.09396910667419434,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.12620492279529572,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.021420922130346298,
|
|
"len_reward/correct/p75": 0.26176321506500244,
|
|
"len_reward/correct/var": 0.05913642793893814,
|
|
"len_reward/incorrect": -0.15916328132152557,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.10764012485742569,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.28188976645469666,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03061472624540329,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.0,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.1725209653377533,
|
|
"len_reward/p75": 0.17705444991588593,
|
|
"len_reward/var": 0.0632537305355072,
|
|
"loss": -0.6198,
|
|
"mean_logprobs": -0.16796875,
|
|
"mean_logprobs/var": 0.01263427734375,
|
|
"missing_response_penalty": -0.2291666716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.602739691734314,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.24277016520500183,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.17757420241832733,
|
|
"non_smoothed_logprobs/min": -16.875,
|
|
"num_completions/total": 384,
|
|
"policy_loss": -0.6197916865348816,
|
|
"policy_loss/max": 0.0,
|
|
"policy_loss/median": -1.0,
|
|
"policy_loss/min": -1.0,
|
|
"policy_loss/p25": -1.0,
|
|
"policy_loss/p75": 0.0,
|
|
"policy_loss/var": 0.2368837296962738,
|
|
"pred_value_log_probs": -0.705960750579834,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.7578125,
|
|
"pred_value_log_probs/min": -1.46875,
|
|
"pred_value_log_probs/p25": -0.7578125,
|
|
"pred_value_log_probs/p75": -0.7578125,
|
|
"pred_value_log_probs/var": 0.0509880967438221,
|
|
"reward": 0.6197916865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.2368837296962738,
|
|
"rewards/accuracy_reward": 0.6197916865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.2368837296962738,
|
|
"rewards/answer_logprob_reward": 2.6923842430114746,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.0986123085021973,
|
|
"rewards/answer_logprob_reward/p25": 2.079441547393799,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.593589186668396,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": -0.0022739358246326447,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.0,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.1725209653377533,
|
|
"rewards/len_reward/p75": 0.17705444991588593,
|
|
"rewards/len_reward/var": 0.0632537305355072,
|
|
"rewards/missing_response_penalty": -0.2291666716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.17757420241832733,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 2,
|
|
"value_loss": 2.7421875,
|
|
"values": -0.7263092994689941,
|
|
"values/max": 0.6708984375,
|
|
"values/median": -0.6884765625,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.6884765625,
|
|
"values/p75": -0.6884765625,
|
|
"values/var": 0.1137716993689537
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 2.1276326151564717e-05,
|
|
"adam_stats/lr_effective_mean": 7.12119209086981e-12,
|
|
"adam_stats/lr_effective_min": -2.1276304323691875e-05,
|
|
"adam_stats/m_t_max": 0.0102612329646945,
|
|
"adam_stats/m_t_mean": -4.9798998258410165e-11,
|
|
"adam_stats/m_t_min": -0.007576906122267246,
|
|
"adam_stats/v_t_max": 1.2175996744190343e-05,
|
|
"adam_stats/v_t_mean": 3.826528937461715e-13,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.05063291139240506,
|
|
"eval_accuracy_reward": 0.5266667008399963,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.25012266635894775,
|
|
"eval_advantages": 0.5266667008399963,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.25012266635894775,
|
|
"eval_all_logprobs": -0.1727336347103119,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": -4.0531158447265625e-05,
|
|
"eval_all_logprobs/min": -12.5,
|
|
"eval_all_logprobs/p25": -0.0234375,
|
|
"eval_all_logprobs/p75": -7.152557373046875e-07,
|
|
"eval_all_logprobs/var": 0.31189486384391785,
|
|
"eval_answer_logprob_reward": 2.7104697227478027,
|
|
"eval_answer_logprob_reward/correct": 3.166231870651245,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.03393346816301346,
|
|
"eval_answer_logprob_reward/incorrect": 2.2033538818359375,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/incorrect/min": 0.9162907600402832,
|
|
"eval_answer_logprob_reward/incorrect/p25": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.9209528565406799,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 0.9162907600402832,
|
|
"eval_answer_logprob_reward/p25": 2.4243624210357666,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.6840105056762695,
|
|
"eval_completion_length": 598.6400146484375,
|
|
"eval_completion_length/correct": 550.6582641601562,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 492.0,
|
|
"eval_completion_length/correct/min": 103.0,
|
|
"eval_completion_length/correct/p25": 349.0,
|
|
"eval_completion_length/correct/p75": 736.75,
|
|
"eval_completion_length/correct/var": 68220.328125,
|
|
"eval_completion_length/incorrect": 652.0281372070312,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 661.0,
|
|
"eval_completion_length/incorrect/min": 14.0,
|
|
"eval_completion_length/incorrect/p25": 342.5,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 132255.953125,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 542.0,
|
|
"eval_completion_length/min": 14.0,
|
|
"eval_completion_length/p25": 347.0,
|
|
"eval_completion_length/p75": 990.25,
|
|
"eval_completion_length/var": 100759.7890625,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 352667.0,
|
|
"eval_len_reward": -0.0232350155711174,
|
|
"eval_len_reward/correct": 0.08221171796321869,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.09401709586381912,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.04879438504576683,
|
|
"eval_len_reward/correct/p75": 0.2662348449230194,
|
|
"eval_len_reward/correct/var": 0.0583944208920002,
|
|
"eval_len_reward/incorrect": -0.14056305587291718,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.04492584988474846,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2636667490005493,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.028694583103060722,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.19380633533000946,
|
|
"eval_len_reward/p75": 0.10665474832057953,
|
|
"eval_len_reward/var": 0.05660673975944519,
|
|
"eval_mean_logprobs": -0.208984375,
|
|
"eval_mean_logprobs/var": 0.03466796875,
|
|
"eval_missing_response_penalty": -0.25,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.5281689763069153,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": -1.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.2509739398956299,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": -0.25,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.1881270706653595,
|
|
"eval_num_completions/total": 600,
|
|
"eval_pred_value_log_probs": -0.7176648378372192,
|
|
"eval_pred_value_log_probs/max": -2.2649765014648438e-06,
|
|
"eval_pred_value_log_probs/median": -0.76171875,
|
|
"eval_pred_value_log_probs/min": -1.171875,
|
|
"eval_pred_value_log_probs/p25": -0.76171875,
|
|
"eval_pred_value_log_probs/p75": -0.76171875,
|
|
"eval_pred_value_log_probs/var": 0.02239096909761429,
|
|
"eval_reward": 0.5266667008399963,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.25012266635894775,
|
|
"eval_rewards/accuracy_reward": 0.5266667008399963,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.25012266635894775,
|
|
"eval_rewards/answer_logprob_reward": 2.7104697227478027,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 0.9162907600402832,
|
|
"eval_rewards/answer_logprob_reward/p25": 2.4243624210357666,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.6840105056762695,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": -0.0232350155711174,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.19380633533000946,
|
|
"eval_rewards/len_reward/p75": 0.10665474832057953,
|
|
"eval_rewards/len_reward/var": 0.05660673975944519,
|
|
"eval_rewards/missing_response_penalty": -0.25,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": -0.25,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.1881270706653595,
|
|
"eval_runtime": 171.3262,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.7187559008598328,
|
|
"eval_values/max": 1.9427032470703125,
|
|
"eval_values/median": -0.68359375,
|
|
"eval_values/min": -1.9771080017089844,
|
|
"eval_values/p25": -0.68359375,
|
|
"eval_values/p75": -0.68359375,
|
|
"eval_values/var": 0.09229511767625809,
|
|
"step": 2
|
|
},
|
|
{
|
|
"accuracy_reward": 0.5885416865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.24342823028564453,
|
|
"adam_stats/lr_effective_max": 3.7253634218359366e-05,
|
|
"adam_stats/lr_effective_mean": -6.315423184410918e-11,
|
|
"adam_stats/lr_effective_min": -3.725680653587915e-05,
|
|
"adam_stats/m_t_max": 0.012103762477636337,
|
|
"adam_stats/m_t_mean": -6.582498435214745e-11,
|
|
"adam_stats/m_t_min": -0.009285036474466324,
|
|
"adam_stats/v_t_max": 1.2986727597308345e-05,
|
|
"adam_stats/v_t_mean": 4.846660182564189e-13,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.5885416865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.24342823028564453,
|
|
"all_logprobs": -0.1633242964744568,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": -2.7060508728027344e-05,
|
|
"all_logprobs/min": -10.5625,
|
|
"all_logprobs/p25": -0.0196533203125,
|
|
"all_logprobs/p75": -4.76837158203125e-07,
|
|
"all_logprobs/var": 0.2853495478630066,
|
|
"answer_logprob_reward": 2.680971622467041,
|
|
"answer_logprob_reward/correct": 3.067641019821167,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06491584330797195,
|
|
"answer_logprob_reward/incorrect": 2.1278867721557617,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/min": 0.9808292388916016,
|
|
"answer_logprob_reward/incorrect/p25": 1.0986123085021973,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.8952284455299377,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 0.9808292388916016,
|
|
"answer_logprob_reward/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.618637204170227,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 593.7916870117188,
|
|
"completion_length/correct": 532.1327514648438,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 474.0,
|
|
"completion_length/correct/min": 75.0,
|
|
"completion_length/correct/p25": 332.0,
|
|
"completion_length/correct/p75": 682.0,
|
|
"completion_length/correct/var": 69216.390625,
|
|
"completion_length/incorrect": 681.9873657226562,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1006.0,
|
|
"completion_length/incorrect/min": 11.0,
|
|
"completion_length/incorrect/p25": 315.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 152385.125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 521.0,
|
|
"completion_length/min": 11.0,
|
|
"completion_length/p25": 324.75,
|
|
"completion_length/p75": 1024.0,
|
|
"completion_length/var": 108284.703125,
|
|
"epoch": 0.0759493670886076,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 349355.0,
|
|
"grad_norm": 0.49451473355293274,
|
|
"kl": -2.2590160369873047e-05,
|
|
"learning_rate": 7.500000000000001e-06,
|
|
"len_reward": -0.018583837896585464,
|
|
"len_reward/correct": 0.08518543094396591,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.13620124757289886,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.09035760164260864,
|
|
"len_reward/correct/p75": 0.24853037297725677,
|
|
"len_reward/correct/var": 0.05470568314194679,
|
|
"len_reward/incorrect": -0.1670132875442505,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.11991434544324875,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.2974925935268402,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03210582956671715,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.0,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.17814569175243378,
|
|
"len_reward/p75": 0.17193058133125305,
|
|
"len_reward/var": 0.0606730654835701,
|
|
"loss": -0.5885,
|
|
"mean_logprobs": -0.1953125,
|
|
"mean_logprobs/var": 0.0274658203125,
|
|
"missing_response_penalty": -0.2291666716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.5569620132446289,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2499188333749771,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.17757417261600494,
|
|
"non_smoothed_logprobs/min": -16.6875,
|
|
"num_completions/total": 576,
|
|
"policy_loss": -0.5885425806045532,
|
|
"policy_loss/max": 4.304658432374708e-06,
|
|
"policy_loss/median": -1.0,
|
|
"policy_loss/min": -1.0000050067901611,
|
|
"policy_loss/p25": -1.0000008344650269,
|
|
"policy_loss/p75": -1.2218952178955078e-06,
|
|
"policy_loss/var": 0.24342793226242065,
|
|
"pred_value_log_probs": -0.7097456455230713,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.76171875,
|
|
"pred_value_log_probs/min": -1.1796875,
|
|
"pred_value_log_probs/p25": -0.76171875,
|
|
"pred_value_log_probs/p75": -0.76171875,
|
|
"pred_value_log_probs/var": 0.03592011332511902,
|
|
"reward": 0.5885416865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.24342823028564453,
|
|
"rewards/accuracy_reward": 0.5885416865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.24342823028564453,
|
|
"rewards/answer_logprob_reward": 2.680971622467041,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 0.9808292388916016,
|
|
"rewards/answer_logprob_reward/p25": 2.4849066734313965,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.618637204170227,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": -0.018583837896585464,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.0,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.17814569175243378,
|
|
"rewards/len_reward/p75": 0.17193058133125305,
|
|
"rewards/len_reward/var": 0.0606730654835701,
|
|
"rewards/missing_response_penalty": -0.2291666716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.17757417261600494,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 3,
|
|
"value_loss": 2.5546875,
|
|
"values": -0.6995077133178711,
|
|
"values/max": 2.000021457672119,
|
|
"values/median": -0.68359375,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.68359375,
|
|
"values/p75": -0.68359375,
|
|
"values/var": 0.1517854928970337
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 3.7253634218359366e-05,
|
|
"adam_stats/lr_effective_mean": -6.315423184410918e-11,
|
|
"adam_stats/lr_effective_min": -3.725680653587915e-05,
|
|
"adam_stats/m_t_max": 0.012103762477636337,
|
|
"adam_stats/m_t_mean": -6.582498435214745e-11,
|
|
"adam_stats/m_t_min": -0.009285036474466324,
|
|
"adam_stats/v_t_max": 1.2986727597308345e-05,
|
|
"adam_stats/v_t_mean": 4.846660182564189e-13,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.0759493670886076,
|
|
"eval_accuracy_reward": 0.5133333206176758,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2506577670574188,
|
|
"eval_advantages": 0.5133333206176758,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2506577670574188,
|
|
"eval_all_logprobs": -0.15390874445438385,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": -2.1457672119140625e-05,
|
|
"eval_all_logprobs/min": -13.25,
|
|
"eval_all_logprobs/p25": -0.01416015625,
|
|
"eval_all_logprobs/p75": -4.76837158203125e-07,
|
|
"eval_all_logprobs/var": 0.2686237096786499,
|
|
"eval_answer_logprob_reward": 2.7141823768615723,
|
|
"eval_answer_logprob_reward/correct": 3.101086139678955,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07403698563575745,
|
|
"eval_answer_logprob_reward/incorrect": 2.3060784339904785,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/min": 0.9162907600402832,
|
|
"eval_answer_logprob_reward/incorrect/p25": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.8032733798027039,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 0.9162907600402832,
|
|
"eval_answer_logprob_reward/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.5858576893806458,
|
|
"eval_completion_length": 588.5166625976562,
|
|
"eval_completion_length/correct": 532.7012939453125,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 453.0,
|
|
"eval_completion_length/correct/min": 20.0,
|
|
"eval_completion_length/correct/p25": 345.5,
|
|
"eval_completion_length/correct/p75": 690.25,
|
|
"eval_completion_length/correct/var": 65450.05859375,
|
|
"eval_completion_length/incorrect": 647.390380859375,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 698.0,
|
|
"eval_completion_length/incorrect/min": 16.0,
|
|
"eval_completion_length/incorrect/p25": 330.5,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 135004.890625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 509.0,
|
|
"eval_completion_length/min": 16.0,
|
|
"eval_completion_length/p25": 336.75,
|
|
"eval_completion_length/p75": 960.75,
|
|
"eval_completion_length/var": 102258.8125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 529222.0,
|
|
"eval_len_reward": -0.02000892348587513,
|
|
"eval_len_reward/correct": 0.09573789685964584,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.14518091082572937,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.049733877182006836,
|
|
"eval_len_reward/correct/p75": 0.2577993869781494,
|
|
"eval_len_reward/correct/var": 0.0548153854906559,
|
|
"eval_len_reward/incorrect": -0.14209803938865662,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.040425531566143036,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.29070523381233215,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03147710859775543,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.17550882697105408,
|
|
"eval_len_reward/p75": 0.1517728716135025,
|
|
"eval_len_reward/var": 0.05749284476041794,
|
|
"eval_mean_logprobs": -0.185546875,
|
|
"eval_mean_logprobs/var": 0.0242919921875,
|
|
"eval_missing_response_penalty": -0.23000000417232513,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.4726027250289917,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.2509683668613434,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.17769230902194977,
|
|
"eval_num_completions/total": 900,
|
|
"eval_pred_value_log_probs": -0.7149859666824341,
|
|
"eval_pred_value_log_probs/max": -2.1457672119140625e-06,
|
|
"eval_pred_value_log_probs/median": -0.7578125,
|
|
"eval_pred_value_log_probs/min": -1.1953125,
|
|
"eval_pred_value_log_probs/p25": -0.7578125,
|
|
"eval_pred_value_log_probs/p75": -0.7578125,
|
|
"eval_pred_value_log_probs/var": 0.022274916991591454,
|
|
"eval_reward": 0.5133333206176758,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2506577670574188,
|
|
"eval_rewards/accuracy_reward": 0.5133333206176758,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2506577670574188,
|
|
"eval_rewards/answer_logprob_reward": 2.7141823768615723,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 0.9162907600402832,
|
|
"eval_rewards/answer_logprob_reward/p25": 2.525728702545166,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.5858576893806458,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": -0.02000892348587513,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.17550882697105408,
|
|
"eval_rewards/len_reward/p75": 0.1517728716135025,
|
|
"eval_rewards/len_reward/var": 0.05749284476041794,
|
|
"eval_rewards/missing_response_penalty": -0.23000000417232513,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.17769230902194977,
|
|
"eval_runtime": 174.2124,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.720794141292572,
|
|
"eval_values/max": 1.951904296875,
|
|
"eval_values/median": -0.6875,
|
|
"eval_values/min": -1.9825439453125,
|
|
"eval_values/p25": -0.6875,
|
|
"eval_values/p75": -0.6875,
|
|
"eval_values/var": 0.09353592246770859,
|
|
"step": 3
|
|
},
|
|
{
|
|
"accuracy_reward": 0.6354166865348816,
|
|
"accuracy_reward/correct": 0.9999999403953552,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.23287522792816162,
|
|
"adam_stats/lr_effective_max": 5.4755189921706915e-05,
|
|
"adam_stats/lr_effective_mean": -2.1790967641432957e-10,
|
|
"adam_stats/lr_effective_min": -5.476594014908187e-05,
|
|
"adam_stats/m_t_max": 0.018315263092517853,
|
|
"adam_stats/m_t_mean": -1.1892935247725944e-10,
|
|
"adam_stats/m_t_min": -0.014240323565900326,
|
|
"adam_stats/v_t_max": 1.8482092855265364e-05,
|
|
"adam_stats/v_t_mean": 6.644819785552036e-13,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.6354166865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.23287522792816162,
|
|
"all_logprobs": -0.1348327398300171,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": -8.106231689453125e-06,
|
|
"all_logprobs/min": -9.8125,
|
|
"all_logprobs/p25": -0.0068359375,
|
|
"all_logprobs/p75": -1.1920928955078125e-07,
|
|
"all_logprobs/var": 0.22426800429821014,
|
|
"answer_logprob_reward": 2.72711181640625,
|
|
"answer_logprob_reward/correct": 3.032691717147827,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 1.7917594909667969,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.11573203653097153,
|
|
"answer_logprob_reward/incorrect": 2.194528341293335,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 1.5686159133911133,
|
|
"answer_logprob_reward/incorrect/min": 1.2321436405181885,
|
|
"answer_logprob_reward/incorrect/p25": 1.3862943649291992,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.7170899510383606,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.2321436405181885,
|
|
"answer_logprob_reward/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.49596965312957764,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 606.4427490234375,
|
|
"completion_length/correct": 553.5901489257812,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 483.0,
|
|
"completion_length/correct/min": 100.0,
|
|
"completion_length/correct/p25": 344.5,
|
|
"completion_length/correct/p75": 748.75,
|
|
"completion_length/correct/var": 73280.8515625,
|
|
"completion_length/incorrect": 698.55712890625,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 883.0,
|
|
"completion_length/incorrect/min": 11.0,
|
|
"completion_length/incorrect/p25": 339.75,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 143602.21875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 563.0,
|
|
"completion_length/min": 11.0,
|
|
"completion_length/p25": 342.0,
|
|
"completion_length/p75": 1024.0,
|
|
"completion_length/var": 103195.2109375,
|
|
"epoch": 0.10126582278481013,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 465792.0,
|
|
"grad_norm": 0.6282712817192078,
|
|
"kl": 3.647804260253906e-05,
|
|
"learning_rate": 1e-05,
|
|
"len_reward": 0.0003473448450677097,
|
|
"len_reward/correct": 0.08404155820608139,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.15213315188884735,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.06679219007492065,
|
|
"len_reward/correct/p75": 0.257729709148407,
|
|
"len_reward/correct/var": 0.06340916454792023,
|
|
"len_reward/incorrect": -0.14551971852779388,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1081407368183136,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.23544758558273315,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.025769773870706558,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.0,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.16495808959007263,
|
|
"len_reward/p75": 0.20012810826301575,
|
|
"len_reward/var": 0.061751849949359894,
|
|
"loss": -0.6354,
|
|
"mean_logprobs": -0.1513671875,
|
|
"mean_logprobs/var": 0.01495361328125,
|
|
"missing_response_penalty": -0.203125,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.5571428537368774,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2503105401992798,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.1627127081155777,
|
|
"non_smoothed_logprobs/min": -16.8125,
|
|
"num_completions/total": 768,
|
|
"policy_loss": -0.6354151964187622,
|
|
"policy_loss/max": 2.843683432729449e-05,
|
|
"policy_loss/median": -0.9999978542327881,
|
|
"policy_loss/min": -1.0000046491622925,
|
|
"policy_loss/p25": -0.9999995231628418,
|
|
"policy_loss/p75": 0.0,
|
|
"policy_loss/var": 0.23287516832351685,
|
|
"pred_value_log_probs": -0.7148666381835938,
|
|
"pred_value_log_probs/max": -0.00012493133544921875,
|
|
"pred_value_log_probs/median": -0.7578125,
|
|
"pred_value_log_probs/min": -1.203125,
|
|
"pred_value_log_probs/p25": -0.7578125,
|
|
"pred_value_log_probs/p75": -0.7578125,
|
|
"pred_value_log_probs/var": 0.024815544486045837,
|
|
"reward": 0.6354166865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.23287522792816162,
|
|
"rewards/accuracy_reward": 0.6354166865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.23287522792816162,
|
|
"rewards/answer_logprob_reward": 2.72711181640625,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.2321436405181885,
|
|
"rewards/answer_logprob_reward/p25": 2.4849066734313965,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.49596965312957764,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.0003473448450677097,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.0,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.16495808959007263,
|
|
"rewards/len_reward/p75": 0.20012810826301575,
|
|
"rewards/len_reward/var": 0.061751849949359894,
|
|
"rewards/missing_response_penalty": -0.203125,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.1627127081155777,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 4,
|
|
"value_loss": 2.390625,
|
|
"values": -0.6947605013847351,
|
|
"values/max": 1.9998908042907715,
|
|
"values/median": -0.6875,
|
|
"values/min": -1.89459228515625,
|
|
"values/p25": -0.6875,
|
|
"values/p75": -0.6875,
|
|
"values/var": 0.1420281082391739
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 5.4755189921706915e-05,
|
|
"adam_stats/lr_effective_mean": -2.1790967641432957e-10,
|
|
"adam_stats/lr_effective_min": -5.476594014908187e-05,
|
|
"adam_stats/m_t_max": 0.018315263092517853,
|
|
"adam_stats/m_t_mean": -1.1892935247725944e-10,
|
|
"adam_stats/m_t_min": -0.014240323565900326,
|
|
"adam_stats/v_t_max": 1.8482092855265364e-05,
|
|
"adam_stats/v_t_mean": 6.644819785552036e-13,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.10126582278481013,
|
|
"eval_accuracy_reward": 0.5633333325386047,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.24681159853935242,
|
|
"eval_advantages": 0.5633333325386047,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.24681159853935242,
|
|
"eval_all_logprobs": -0.10831712186336517,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": -2.384185791015625e-06,
|
|
"eval_all_logprobs/min": -11.125,
|
|
"eval_all_logprobs/p25": -0.002197265625,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.17934179306030273,
|
|
"eval_answer_logprob_reward": 2.797687530517578,
|
|
"eval_answer_logprob_reward/correct": 3.1648600101470947,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.040031738579273224,
|
|
"eval_answer_logprob_reward/incorrect": 2.3240067958831787,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.021651268005371,
|
|
"eval_answer_logprob_reward/incorrect/p25": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.8312591910362244,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.021651268005371,
|
|
"eval_answer_logprob_reward/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.5584139823913574,
|
|
"eval_completion_length": 631.703369140625,
|
|
"eval_completion_length/correct": 575.7633056640625,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 518.0,
|
|
"eval_completion_length/correct/min": 126.0,
|
|
"eval_completion_length/correct/p25": 363.0,
|
|
"eval_completion_length/correct/p75": 784.0,
|
|
"eval_completion_length/correct/var": 71524.703125,
|
|
"eval_completion_length/incorrect": 703.8702392578125,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 803.0,
|
|
"eval_completion_length/incorrect/min": 24.0,
|
|
"eval_completion_length/incorrect/p25": 410.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 115899.25,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 586.0,
|
|
"eval_completion_length/min": 24.0,
|
|
"eval_completion_length/p25": 377.5,
|
|
"eval_completion_length/p75": 1024.0,
|
|
"eval_completion_length/var": 94629.28125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 718733.0,
|
|
"eval_len_reward": -0.01611892506480217,
|
|
"eval_len_reward/correct": 0.07804737985134125,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1132001057267189,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.07658227533102036,
|
|
"eval_len_reward/correct/p75": 0.26623716950416565,
|
|
"eval_len_reward/correct/var": 0.06207133084535599,
|
|
"eval_len_reward/incorrect": -0.13760064542293549,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.06359031796455383,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.24769610166549683,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.02720070257782936,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.1695822775363922,
|
|
"eval_len_reward/p75": 0.14719371497631073,
|
|
"eval_len_reward/var": 0.05818033963441849,
|
|
"eval_mean_logprobs": -0.130859375,
|
|
"eval_mean_logprobs/var": 0.0185546875,
|
|
"eval_missing_response_penalty": -0.21666666865348816,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.49618321657180786,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.25190839171409607,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.17028985917568207,
|
|
"eval_num_completions/total": 1200,
|
|
"eval_pred_value_log_probs": -0.7214562892913818,
|
|
"eval_pred_value_log_probs/max": -1.7881393432617188e-06,
|
|
"eval_pred_value_log_probs/median": -0.765625,
|
|
"eval_pred_value_log_probs/min": -1.078125,
|
|
"eval_pred_value_log_probs/p25": -0.765625,
|
|
"eval_pred_value_log_probs/p75": -0.765625,
|
|
"eval_pred_value_log_probs/var": 0.023659951984882355,
|
|
"eval_reward": 0.5633333325386047,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.24681159853935242,
|
|
"eval_rewards/accuracy_reward": 0.5633333325386047,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.24681159853935242,
|
|
"eval_rewards/answer_logprob_reward": 2.797687530517578,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.021651268005371,
|
|
"eval_rewards/answer_logprob_reward/p25": 2.525728702545166,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.5584139823913574,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": -0.01611892506480217,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.1695822775363922,
|
|
"eval_rewards/len_reward/p75": 0.14719371497631073,
|
|
"eval_rewards/len_reward/var": 0.05818033963441849,
|
|
"eval_rewards/missing_response_penalty": -0.21666666865348816,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.17028985917568207,
|
|
"eval_runtime": 177.3334,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.7129337787628174,
|
|
"eval_values/max": 1.9513092041015625,
|
|
"eval_values/median": -0.6796875,
|
|
"eval_values/min": -1.9838800430297852,
|
|
"eval_values/p25": -0.6796875,
|
|
"eval_values/p75": -0.6796875,
|
|
"eval_values/var": 0.09706096351146698,
|
|
"step": 4
|
|
},
|
|
{
|
|
"accuracy_reward": 0.6927083730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.21397797763347626,
|
|
"adam_stats/lr_effective_max": 5.835445335833356e-05,
|
|
"adam_stats/lr_effective_mean": 2.061619069904097e-10,
|
|
"adam_stats/lr_effective_min": -5.8340159739600495e-05,
|
|
"adam_stats/m_t_max": 0.011445377953350544,
|
|
"adam_stats/m_t_mean": -6.51839693333045e-11,
|
|
"adam_stats/m_t_min": -0.007816392928361893,
|
|
"adam_stats/v_t_max": 2.100208257616032e-05,
|
|
"adam_stats/v_t_mean": 1.1845901863594133e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.6927083730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.21397797763347626,
|
|
"all_logprobs": -0.11215102672576904,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": -2.0265579223632812e-06,
|
|
"all_logprobs/min": -12.375,
|
|
"all_logprobs/p25": -0.0023345947265625,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.189283087849617,
|
|
"answer_logprob_reward": 2.941647529602051,
|
|
"answer_logprob_reward/correct": 3.1207261085510254,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.03672637790441513,
|
|
"answer_logprob_reward/incorrect": 2.5379624366760254,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.5686159133911133,
|
|
"answer_logprob_reward/incorrect/p25": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.4904671311378479,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.5686159133911133,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.2469889521598816,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 601.8698120117188,
|
|
"completion_length/correct": 492.4436340332031,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 409.0,
|
|
"completion_length/correct/min": 157.0,
|
|
"completion_length/correct/p25": 309.0,
|
|
"completion_length/correct/p75": 635.0,
|
|
"completion_length/correct/var": 57041.28515625,
|
|
"completion_length/incorrect": 848.5423583984375,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 10.0,
|
|
"completion_length/incorrect/p25": 730.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 80482.3515625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 549.0,
|
|
"completion_length/min": 10.0,
|
|
"completion_length/p25": 345.0,
|
|
"completion_length/p75": 947.0,
|
|
"completion_length/var": 90994.625,
|
|
"epoch": 0.12658227848101267,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 581351.0,
|
|
"grad_norm": 2.539113759994507,
|
|
"kl": 0.0005397796630859375,
|
|
"learning_rate": 9.979871469976197e-06,
|
|
"len_reward": 0.011436531320214272,
|
|
"len_reward/correct": 0.11529415845870972,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.16104553639888763,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.0,
|
|
"len_reward/correct/p75": 0.2899543344974518,
|
|
"len_reward/correct/var": 0.05602065846323967,
|
|
"len_reward/incorrect": -0.22268320620059967,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.18915121257305145,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.40228500962257385,
|
|
"len_reward/incorrect/p75": -0.030559808015823364,
|
|
"len_reward/incorrect/var": 0.034158919006586075,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.019392916932702065,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.15810225903987885,
|
|
"len_reward/p75": 0.21998034417629242,
|
|
"len_reward/var": 0.0735311433672905,
|
|
"loss": -0.6927,
|
|
"mean_logprobs": -0.126953125,
|
|
"mean_logprobs/var": 0.0135498046875,
|
|
"missing_response_penalty": -0.1458333432674408,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.47457626461982727,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2536528408527374,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.12521816790103912,
|
|
"non_smoothed_logprobs/min": -16.4375,
|
|
"num_completions/total": 960,
|
|
"policy_loss": -0.6926867365837097,
|
|
"policy_loss/max": 0.0002189636288676411,
|
|
"policy_loss/median": -0.999973714351654,
|
|
"policy_loss/min": -0.9999991059303284,
|
|
"policy_loss/p25": -0.9999872446060181,
|
|
"policy_loss/p75": 4.314817488193512e-06,
|
|
"policy_loss/var": 0.213976189494133,
|
|
"pred_value_log_probs": -0.6869064569473267,
|
|
"pred_value_log_probs/max": -0.0019989013671875,
|
|
"pred_value_log_probs/median": -0.765625,
|
|
"pred_value_log_probs/min": -1.4453125,
|
|
"pred_value_log_probs/p25": -0.765625,
|
|
"pred_value_log_probs/p75": -0.765625,
|
|
"pred_value_log_probs/var": 0.052704740315675735,
|
|
"reward": 0.6927083730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.21397797763347626,
|
|
"rewards/accuracy_reward": 0.6927083730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.21397797763347626,
|
|
"rewards/answer_logprob_reward": 2.941647529602051,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.5686159133911133,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.2469889521598816,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.011436531320214272,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.019392916932702065,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.15810225903987885,
|
|
"rewards/len_reward/p75": 0.21998034417629242,
|
|
"rewards/len_reward/var": 0.0735311433672905,
|
|
"rewards/missing_response_penalty": -0.1458333432674408,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.12521816790103912,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 5,
|
|
"value_loss": 2.6484375,
|
|
"values": -0.7231578826904297,
|
|
"values/max": 1.9903125762939453,
|
|
"values/median": -0.6796875,
|
|
"values/min": -1.866668701171875,
|
|
"values/p25": -0.6796875,
|
|
"values/p75": -0.6796875,
|
|
"values/var": 0.25189828872680664
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 5.835445335833356e-05,
|
|
"adam_stats/lr_effective_mean": 2.061619069904097e-10,
|
|
"adam_stats/lr_effective_min": -5.8340159739600495e-05,
|
|
"adam_stats/m_t_max": 0.011445377953350544,
|
|
"adam_stats/m_t_mean": -6.51839693333045e-11,
|
|
"adam_stats/m_t_min": -0.007816392928361893,
|
|
"adam_stats/v_t_max": 2.100208257616032e-05,
|
|
"adam_stats/v_t_mean": 1.1845901863594133e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.12658227848101267,
|
|
"eval_accuracy_reward": 0.596666693687439,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.24146044254302979,
|
|
"eval_advantages": 0.596666693687439,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.24146044254302979,
|
|
"eval_all_logprobs": -0.0702953189611435,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/min": -16.25,
|
|
"eval_all_logprobs/p25": -0.00011110305786132812,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.11423061788082123,
|
|
"eval_answer_logprob_reward": 2.964020013809204,
|
|
"eval_answer_logprob_reward/correct": 3.1385061740875244,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.057033028453588486,
|
|
"eval_answer_logprob_reward/incorrect": 2.7058961391448975,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.021651268005371,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.5137178301811218,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.021651268005371,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.2853167951107025,
|
|
"eval_completion_length": 695.0266723632812,
|
|
"eval_completion_length/correct": 613.7988891601562,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 555.0,
|
|
"eval_completion_length/correct/min": 12.0,
|
|
"eval_completion_length/correct/p25": 363.0,
|
|
"eval_completion_length/correct/p75": 905.0,
|
|
"eval_completion_length/correct/var": 84074.828125,
|
|
"eval_completion_length/incorrect": 815.1900634765625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 1024.0,
|
|
"eval_completion_length/incorrect/min": 19.0,
|
|
"eval_completion_length/incorrect/p25": 611.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 70829.8515625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 689.0,
|
|
"eval_completion_length/min": 12.0,
|
|
"eval_completion_length/p25": 419.5,
|
|
"eval_completion_length/p75": 1024.0,
|
|
"eval_completion_length/var": 88271.1875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 927241.0,
|
|
"eval_len_reward": -0.01350005529820919,
|
|
"eval_len_reward/correct": 0.08121874183416367,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.15369127690792084,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.13007400929927826,
|
|
"eval_len_reward/correct/p75": 0.3086293339729309,
|
|
"eval_len_reward/correct/var": 0.08572941273450851,
|
|
"eval_len_reward/incorrect": -0.15362125635147095,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.08638916164636612,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.275522381067276,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.02721996046602726,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.21404384076595306,
|
|
"eval_len_reward/p75": 0.21088477969169617,
|
|
"eval_len_reward/var": 0.07527714222669601,
|
|
"eval_mean_logprobs": -0.0732421875,
|
|
"eval_mean_logprobs/var": 0.00927734375,
|
|
"eval_missing_response_penalty": -0.13333334028720856,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.3305785059928894,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.22314050793647766,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.11594202369451523,
|
|
"eval_num_completions/total": 1500,
|
|
"eval_pred_value_log_probs": -0.7040350437164307,
|
|
"eval_pred_value_log_probs/max": -2.384185791015625e-06,
|
|
"eval_pred_value_log_probs/median": -0.74609375,
|
|
"eval_pred_value_log_probs/min": -1.1015625,
|
|
"eval_pred_value_log_probs/p25": -0.74609375,
|
|
"eval_pred_value_log_probs/p75": -0.74609375,
|
|
"eval_pred_value_log_probs/var": 0.023306304588913918,
|
|
"eval_reward": 0.596666693687439,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.24146044254302979,
|
|
"eval_rewards/accuracy_reward": 0.596666693687439,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.24146044254302979,
|
|
"eval_rewards/answer_logprob_reward": 2.964020013809204,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.021651268005371,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.2853167951107025,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": -0.01350005529820919,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.21404384076595306,
|
|
"eval_rewards/len_reward/p75": 0.21088477969169617,
|
|
"eval_rewards/len_reward/var": 0.07527714222669601,
|
|
"eval_rewards/missing_response_penalty": -0.13333334028720856,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.11594202369451523,
|
|
"eval_runtime": 169.8057,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.7374175786972046,
|
|
"eval_values/max": 1.9597244262695312,
|
|
"eval_values/median": -0.70751953125,
|
|
"eval_values/min": -1.983614444732666,
|
|
"eval_values/p25": -0.70751953125,
|
|
"eval_values/p75": -0.70751953125,
|
|
"eval_values/var": 0.10163413733243942,
|
|
"step": 5
|
|
},
|
|
{
|
|
"accuracy_reward": 0.6822916865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.21790467202663422,
|
|
"adam_stats/lr_effective_max": 6.074368502595462e-05,
|
|
"adam_stats/lr_effective_mean": 3.5626293581891844e-10,
|
|
"adam_stats/lr_effective_min": -6.089979433454573e-05,
|
|
"adam_stats/m_t_max": 0.011484922841191292,
|
|
"adam_stats/m_t_mean": -7.447095717871832e-11,
|
|
"adam_stats/m_t_min": -0.007474206853657961,
|
|
"adam_stats/v_t_max": 2.1121284589753486e-05,
|
|
"adam_stats/v_t_mean": 1.2551746751340853e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.6822916865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.21790467202663422,
|
|
"all_logprobs": -0.06589704751968384,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": -1.1920928955078125e-07,
|
|
"all_logprobs/min": -9.9375,
|
|
"all_logprobs/p25": -5.27501106262207e-05,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.10793641209602356,
|
|
"answer_logprob_reward": 2.9347877502441406,
|
|
"answer_logprob_reward/correct": 3.069532632827759,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.08449158817529678,
|
|
"answer_logprob_reward/incorrect": 2.6454176902770996,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.5686159133911133,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.40474987030029297,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.5686159133911133,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.2238491177558899,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 678.4166870117188,
|
|
"completion_length/correct": 585.5496215820312,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 522.0,
|
|
"completion_length/correct/min": 184.0,
|
|
"completion_length/correct/p25": 366.5,
|
|
"completion_length/correct/p75": 813.5,
|
|
"completion_length/correct/var": 72189.390625,
|
|
"completion_length/incorrect": 877.8524169921875,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 35.0,
|
|
"completion_length/incorrect/p25": 828.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 69250.4296875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 646.0,
|
|
"completion_length/min": 35.0,
|
|
"completion_length/p25": 407.75,
|
|
"completion_length/p75": 1024.0,
|
|
"completion_length/var": 89506.1796875,
|
|
"epoch": 0.1518987341772152,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 711607.0,
|
|
"grad_norm": 0.4030705690383911,
|
|
"kl": 0.002655029296875,
|
|
"learning_rate": 9.91964794299315e-06,
|
|
"len_reward": 0.02160990796983242,
|
|
"len_reward/correct": 0.12393714487552643,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.15581050515174866,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.026404786854982376,
|
|
"len_reward/correct/p75": 0.3236168324947357,
|
|
"len_reward/correct/var": 0.07225022464990616,
|
|
"len_reward/incorrect": -0.19814200699329376,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.18134714663028717,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.34914764761924744,
|
|
"len_reward/incorrect/p75": -0.04173106700181961,
|
|
"len_reward/incorrect/var": 0.028696652501821518,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.0,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.18261007964611053,
|
|
"len_reward/p75": 0.26758527755737305,
|
|
"len_reward/var": 0.08079454302787781,
|
|
"loss": -0.6822,
|
|
"mean_logprobs": -0.064453125,
|
|
"mean_logprobs/var": 0.00128173828125,
|
|
"missing_response_penalty": -0.1302083432674408,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4098360538482666,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.24590164422988892,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.11384708434343338,
|
|
"non_smoothed_logprobs/min": -19.1875,
|
|
"num_completions/total": 1152,
|
|
"policy_loss": -0.6821851134300232,
|
|
"policy_loss/max": 0.000812639482319355,
|
|
"policy_loss/median": -0.9998696446418762,
|
|
"policy_loss/min": -0.9999760389328003,
|
|
"policy_loss/p25": -0.9999115467071533,
|
|
"policy_loss/p75": 5.626119673252106e-05,
|
|
"policy_loss/var": 0.21789757907390594,
|
|
"pred_value_log_probs": -0.6675887107849121,
|
|
"pred_value_log_probs/max": -0.0004329681396484375,
|
|
"pred_value_log_probs/median": -0.74609375,
|
|
"pred_value_log_probs/min": -1.203125,
|
|
"pred_value_log_probs/p25": -0.74609375,
|
|
"pred_value_log_probs/p75": -0.744140625,
|
|
"pred_value_log_probs/var": 0.04631832242012024,
|
|
"reward": 0.6822916865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.21790467202663422,
|
|
"rewards/accuracy_reward": 0.6822916865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.21790467202663422,
|
|
"rewards/answer_logprob_reward": 2.9347877502441406,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.5686159133911133,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.2238491177558899,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.02160990796983242,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.0,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.18261007964611053,
|
|
"rewards/len_reward/p75": 0.26758527755737305,
|
|
"rewards/len_reward/var": 0.08079454302787781,
|
|
"rewards/missing_response_penalty": -0.1302083432674408,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.11384708434343338,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 6,
|
|
"value_loss": 2.6328125,
|
|
"values": -0.7369192838668823,
|
|
"values/max": 1.5349578857421875,
|
|
"values/median": -0.70751953125,
|
|
"values/min": -1.932159423828125,
|
|
"values/p25": -0.70751953125,
|
|
"values/p75": -0.70751953125,
|
|
"values/var": 0.23455187678337097
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 6.074368502595462e-05,
|
|
"adam_stats/lr_effective_mean": 3.5626293581891844e-10,
|
|
"adam_stats/lr_effective_min": -6.089979433454573e-05,
|
|
"adam_stats/m_t_max": 0.011484922841191292,
|
|
"adam_stats/m_t_mean": -7.447095717871832e-11,
|
|
"adam_stats/m_t_min": -0.007474206853657961,
|
|
"adam_stats/v_t_max": 2.1121284589753486e-05,
|
|
"adam_stats/v_t_mean": 1.2551746751340853e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.1518987341772152,
|
|
"eval_accuracy_reward": 0.6533333659172058,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22724635899066925,
|
|
"eval_advantages": 0.6533333659172058,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22724635899066925,
|
|
"eval_all_logprobs": -0.04569603130221367,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -8.0625,
|
|
"eval_all_logprobs/p25": -4.291534423828125e-06,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.0705338642001152,
|
|
"eval_answer_logprob_reward": 2.985471248626709,
|
|
"eval_answer_logprob_reward/correct": 3.1265928745269775,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07172445952892303,
|
|
"eval_answer_logprob_reward/incorrect": 2.7195112705230713,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.1394343376159668,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.4924987852573395,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.1394343376159668,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.2540917694568634,
|
|
"eval_completion_length": 595.2000122070312,
|
|
"eval_completion_length/correct": 518.8673095703125,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 476.0,
|
|
"eval_completion_length/correct/min": 98.0,
|
|
"eval_completion_length/correct/p25": 327.5,
|
|
"eval_completion_length/correct/p75": 678.0,
|
|
"eval_completion_length/correct/var": 52477.60546875,
|
|
"eval_completion_length/incorrect": 739.0577392578125,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 752.0,
|
|
"eval_completion_length/incorrect/min": 131.0,
|
|
"eval_completion_length/incorrect/p25": 487.75,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 78506.140625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 549.0,
|
|
"eval_completion_length/min": 98.0,
|
|
"eval_completion_length/p25": 360.0,
|
|
"eval_completion_length/p75": 808.75,
|
|
"eval_completion_length/var": 72286.203125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 1105801.0,
|
|
"eval_len_reward": 0.0503496453166008,
|
|
"eval_len_reward/correct": 0.15384456515312195,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.18005843460559845,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.01752505823969841,
|
|
"eval_len_reward/correct/p75": 0.3630443811416626,
|
|
"eval_len_reward/correct/var": 0.06298115104436874,
|
|
"eval_len_reward/incorrect": -0.14469848573207855,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.06391526758670807,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.27572810649871826,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.030527573078870773,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.12186340987682343,
|
|
"eval_len_reward/p75": 0.2680339813232422,
|
|
"eval_len_reward/var": 0.07184485346078873,
|
|
"eval_mean_logprobs": -0.044921875,
|
|
"eval_mean_logprobs/var": 0.00113677978515625,
|
|
"eval_missing_response_penalty": -0.11666666716337204,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.3365384638309479,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.2254481017589569,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.10340023040771484,
|
|
"eval_num_completions/total": 1800,
|
|
"eval_pred_value_log_probs": -0.6845210790634155,
|
|
"eval_pred_value_log_probs/max": -1.1920928955078125e-07,
|
|
"eval_pred_value_log_probs/median": -0.7265625,
|
|
"eval_pred_value_log_probs/min": -1.0546875,
|
|
"eval_pred_value_log_probs/p25": -0.7265625,
|
|
"eval_pred_value_log_probs/p75": -0.7265625,
|
|
"eval_pred_value_log_probs/var": 0.024145593866705894,
|
|
"eval_reward": 0.6533333659172058,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22724635899066925,
|
|
"eval_rewards/accuracy_reward": 0.6533333659172058,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22724635899066925,
|
|
"eval_rewards/answer_logprob_reward": 2.985471248626709,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.1394343376159668,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.2540917694568634,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.0503496453166008,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.12186340987682343,
|
|
"eval_rewards/len_reward/p75": 0.2680339813232422,
|
|
"eval_rewards/len_reward/var": 0.07184485346078873,
|
|
"eval_rewards/missing_response_penalty": -0.11666666716337204,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.10340023040771484,
|
|
"eval_runtime": 166.4312,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.764191210269928,
|
|
"eval_values/max": 1.9675979614257812,
|
|
"eval_values/median": -0.73388671875,
|
|
"eval_values/min": -1.9830894470214844,
|
|
"eval_values/p25": -0.73388671875,
|
|
"eval_values/p75": -0.73388671875,
|
|
"eval_values/var": 0.1079685240983963,
|
|
"step": 6
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7239583730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.2008889615535736,
|
|
"adam_stats/lr_effective_max": 6.182432116474956e-05,
|
|
"adam_stats/lr_effective_mean": 4.6742158754664587e-11,
|
|
"adam_stats/lr_effective_min": -6.246730481507257e-05,
|
|
"adam_stats/m_t_max": 0.009906132705509663,
|
|
"adam_stats/m_t_mean": -5.941885872218222e-11,
|
|
"adam_stats/m_t_min": -0.006426454987376928,
|
|
"adam_stats/v_t_max": 2.111867979692761e-05,
|
|
"adam_stats/v_t_mean": 1.2852747295274969e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7239583730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.2008889615535736,
|
|
"all_logprobs": -0.04034246504306793,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.375,
|
|
"all_logprobs/p25": -1.6689300537109375e-06,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.0620364248752594,
|
|
"answer_logprob_reward": 2.985330104827881,
|
|
"answer_logprob_reward/correct": 3.119436264038086,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.04714789614081383,
|
|
"answer_logprob_reward/incorrect": 2.6336169242858887,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.3862943649291992,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.5028464794158936,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.3862943649291992,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.2183794379234314,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 587.6927490234375,
|
|
"completion_length/correct": 519.0935668945312,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 472.0,
|
|
"completion_length/correct/min": 163.0,
|
|
"completion_length/correct/p25": 351.5,
|
|
"completion_length/correct/p75": 648.0,
|
|
"completion_length/correct/var": 45573.63671875,
|
|
"completion_length/incorrect": 767.603759765625,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 976.0,
|
|
"completion_length/incorrect/min": 12.0,
|
|
"completion_length/incorrect/p25": 519.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 90514.6640625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 533.0,
|
|
"completion_length/min": 12.0,
|
|
"completion_length/p25": 379.5,
|
|
"completion_length/p75": 777.75,
|
|
"completion_length/var": 69976.6640625,
|
|
"epoch": 0.17721518987341772,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 824444.0,
|
|
"grad_norm": 0.2801690399646759,
|
|
"kl": 0.004913330078125,
|
|
"learning_rate": 9.819814303479268e-06,
|
|
"len_reward": 0.07311590760946274,
|
|
"len_reward/correct": 0.159356489777565,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.19765864312648773,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.005923635791987181,
|
|
"len_reward/correct/p75": 0.3308439254760742,
|
|
"len_reward/correct/var": 0.05828801542520523,
|
|
"len_reward/incorrect": -0.15306219458580017,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.09063062816858292,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.2879432737827301,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.02899157628417015,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.05495617166161537,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.07770079374313354,
|
|
"len_reward/p75": 0.2734633982181549,
|
|
"len_reward/var": 0.0696147084236145,
|
|
"loss": -0.7238,
|
|
"mean_logprobs": -0.04443359375,
|
|
"mean_logprobs/var": 0.006378173828125,
|
|
"missing_response_penalty": -0.1197916716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.43396228551864624,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2503628432750702,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.10599368065595627,
|
|
"non_smoothed_logprobs/min": -19.5625,
|
|
"num_completions/total": 1344,
|
|
"policy_loss": -0.723761796951294,
|
|
"policy_loss/max": 0.00042992449016310275,
|
|
"policy_loss/median": -0.9997755289077759,
|
|
"policy_loss/min": -0.9999435544013977,
|
|
"policy_loss/p25": -0.9998378753662109,
|
|
"policy_loss/p75": 8.520297706127167e-05,
|
|
"policy_loss/var": 0.20088227093219757,
|
|
"pred_value_log_probs": -0.6857487559318542,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.7265625,
|
|
"pred_value_log_probs/min": -1.2109375,
|
|
"pred_value_log_probs/p25": -0.7265625,
|
|
"pred_value_log_probs/p75": -0.7265625,
|
|
"pred_value_log_probs/var": 0.032485608011484146,
|
|
"reward": 0.7239583730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.2008889615535736,
|
|
"rewards/accuracy_reward": 0.7239583730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.2008889615535736,
|
|
"rewards/answer_logprob_reward": 2.985330104827881,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.3862943649291992,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.2183794379234314,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.07311590760946274,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.05495617166161537,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.07770079374313354,
|
|
"rewards/len_reward/p75": 0.2734633982181549,
|
|
"rewards/len_reward/var": 0.0696147084236145,
|
|
"rewards/missing_response_penalty": -0.1197916716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.10599368065595627,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 7,
|
|
"value_loss": 2.8671875,
|
|
"values": -0.7097371816635132,
|
|
"values/max": 2.0,
|
|
"values/median": -0.73388671875,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.73388671875,
|
|
"values/p75": -0.73388671875,
|
|
"values/var": 0.18856659531593323
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 6.182432116474956e-05,
|
|
"adam_stats/lr_effective_mean": 4.6742158754664587e-11,
|
|
"adam_stats/lr_effective_min": -6.246730481507257e-05,
|
|
"adam_stats/m_t_max": 0.009906132705509663,
|
|
"adam_stats/m_t_mean": -5.941885872218222e-11,
|
|
"adam_stats/m_t_min": -0.006426454987376928,
|
|
"adam_stats/v_t_max": 2.111867979692761e-05,
|
|
"adam_stats/v_t_mean": 1.2852747295274969e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.17721518987341772,
|
|
"eval_accuracy_reward": 0.6500000357627869,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2282608449459076,
|
|
"eval_advantages": 0.6500000357627869,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2282608449459076,
|
|
"eval_all_logprobs": -0.034109558910131454,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -9.625,
|
|
"eval_all_logprobs/p25": -5.960464477539062e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.0519154891371727,
|
|
"eval_answer_logprob_reward": 2.9738736152648926,
|
|
"eval_answer_logprob_reward/correct": 3.1169309616088867,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07894976437091827,
|
|
"eval_answer_logprob_reward/incorrect": 2.708195686340332,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.1394343376159668,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.45827212929725647,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.1394343376159668,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.24875830113887787,
|
|
"eval_completion_length": 580.086669921875,
|
|
"eval_completion_length/correct": 498.4154052734375,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 462.0,
|
|
"eval_completion_length/correct/min": 180.0,
|
|
"eval_completion_length/correct/p25": 337.5,
|
|
"eval_completion_length/correct/p75": 625.0,
|
|
"eval_completion_length/correct/var": 42704.66796875,
|
|
"eval_completion_length/incorrect": 731.761962890625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 750.0,
|
|
"eval_completion_length/incorrect/min": 53.0,
|
|
"eval_completion_length/incorrect/p25": 481.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 78969.8046875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 526.0,
|
|
"eval_completion_length/min": 53.0,
|
|
"eval_completion_length/p25": 373.0,
|
|
"eval_completion_length/p75": 771.5,
|
|
"eval_completion_length/var": 67604.7421875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 1279827.0,
|
|
"eval_len_reward": 0.04109089449048042,
|
|
"eval_len_reward/correct": 0.14343681931495667,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1515151560306549,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.0,
|
|
"eval_len_reward/correct/p75": 0.3186544179916382,
|
|
"eval_len_reward/correct/var": 0.053380273282527924,
|
|
"eval_len_reward/incorrect": -0.14898011088371277,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.06081081181764603,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2922284007072449,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03230222314596176,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.09275729954242706,
|
|
"eval_len_reward/p75": 0.23071445524692535,
|
|
"eval_len_reward/var": 0.06538829207420349,
|
|
"eval_mean_logprobs": -0.03369140625,
|
|
"eval_mean_logprobs/var": 0.0013427734375,
|
|
"eval_missing_response_penalty": -0.12666666507720947,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.3619047701358795,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.2331501841545105,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.1109922006726265,
|
|
"eval_num_completions/total": 2100,
|
|
"eval_pred_value_log_probs": -0.6708398461341858,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.71484375,
|
|
"eval_pred_value_log_probs/min": -1.0625,
|
|
"eval_pred_value_log_probs/p25": -0.71484375,
|
|
"eval_pred_value_log_probs/p75": -0.71484375,
|
|
"eval_pred_value_log_probs/var": 0.025487350299954414,
|
|
"eval_reward": 0.6500000357627869,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2282608449459076,
|
|
"eval_rewards/accuracy_reward": 0.6500000357627869,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2282608449459076,
|
|
"eval_rewards/answer_logprob_reward": 2.9738736152648926,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.1394343376159668,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.24875830113887787,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.04109089449048042,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.09275729954242706,
|
|
"eval_rewards/len_reward/p75": 0.23071445524692535,
|
|
"eval_rewards/len_reward/var": 0.06538829207420349,
|
|
"eval_rewards/missing_response_penalty": -0.12666666507720947,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.1109922006726265,
|
|
"eval_runtime": 186.1527,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.7883679270744324,
|
|
"eval_values/max": 1.9758377075195312,
|
|
"eval_values/median": -0.7548828125,
|
|
"eval_values/min": -1.973602294921875,
|
|
"eval_values/p25": -0.7548828125,
|
|
"eval_values/p75": -0.7548828125,
|
|
"eval_values/var": 0.11575037986040115,
|
|
"step": 7
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7760416865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.1747109442949295,
|
|
"adam_stats/lr_effective_max": 6.243507232284173e-05,
|
|
"adam_stats/lr_effective_mean": -1.6910949030002342e-10,
|
|
"adam_stats/lr_effective_min": -6.287322321441025e-05,
|
|
"adam_stats/m_t_max": 0.007395743392407894,
|
|
"adam_stats/m_t_mean": -2.0997197036631832e-11,
|
|
"adam_stats/m_t_min": -0.004758418537676334,
|
|
"adam_stats/v_t_max": 2.132852932845708e-05,
|
|
"adam_stats/v_t_mean": 1.3262481390885483e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7760416865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.1747109442949295,
|
|
"all_logprobs": -0.025388197973370552,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.53125,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.037190306931734085,
|
|
"answer_logprob_reward": 2.963308334350586,
|
|
"answer_logprob_reward/correct": 3.0721981525421143,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.07161752879619598,
|
|
"answer_logprob_reward/incorrect": 2.5859925746917725,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.5686159133911133,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.47603172063827515,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.5686159133911133,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.20147234201431274,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 567.984375,
|
|
"completion_length/correct": 491.16778564453125,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 435.0,
|
|
"completion_length/correct/min": 173.0,
|
|
"completion_length/correct/p25": 325.0,
|
|
"completion_length/correct/p75": 593.0,
|
|
"completion_length/correct/var": 47529.54296875,
|
|
"completion_length/incorrect": 834.1627807617188,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 308.0,
|
|
"completion_length/incorrect/p25": 657.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 65805.8125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 483.0,
|
|
"completion_length/min": 173.0,
|
|
"completion_length/p25": 357.5,
|
|
"completion_length/p75": 758.75,
|
|
"completion_length/var": 71853.5390625,
|
|
"epoch": 0.20253164556962025,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 933497.0,
|
|
"grad_norm": 0.3094690442085266,
|
|
"kl": 0.006195068359375,
|
|
"learning_rate": 9.681174353198687e-06,
|
|
"len_reward": 0.1136961430311203,
|
|
"len_reward/correct": 0.20485958456993103,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2629537582397461,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.10438908636569977,
|
|
"len_reward/correct/p75": 0.3595964014530182,
|
|
"len_reward/correct/var": 0.04752177372574806,
|
|
"len_reward/incorrect": -0.20219576358795166,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.15865178406238556,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.36952483654022217,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03490697592496872,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.16337041556835175,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.06925267726182938,
|
|
"len_reward/p75": 0.32950764894485474,
|
|
"len_reward/var": 0.07344760000705719,
|
|
"loss": -0.7758,
|
|
"mean_logprobs": -0.023193359375,
|
|
"mean_logprobs/var": 0.00034332275390625,
|
|
"missing_response_penalty": -0.109375,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4883720874786377,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.25581395626068115,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.09792213141918182,
|
|
"non_smoothed_logprobs/min": -19.8125,
|
|
"num_completions/total": 1536,
|
|
"policy_loss": -0.7757939100265503,
|
|
"policy_loss/max": 0.0003666747361421585,
|
|
"policy_loss/median": -0.9997168779373169,
|
|
"policy_loss/min": -0.9999092221260071,
|
|
"policy_loss/p25": -0.999791145324707,
|
|
"policy_loss/p75": -0.9995872378349304,
|
|
"policy_loss/var": 0.1746944934129715,
|
|
"pred_value_log_probs": -0.6520019769668579,
|
|
"pred_value_log_probs/max": -0.0028533935546875,
|
|
"pred_value_log_probs/median": -0.71484375,
|
|
"pred_value_log_probs/min": -1.3125,
|
|
"pred_value_log_probs/p25": -0.71484375,
|
|
"pred_value_log_probs/p75": -0.71484375,
|
|
"pred_value_log_probs/var": 0.051762331277132034,
|
|
"reward": 0.7760416865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.1747109442949295,
|
|
"rewards/accuracy_reward": 0.7760416865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.1747109442949295,
|
|
"rewards/answer_logprob_reward": 2.963308334350586,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.5686159133911133,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.20147234201431274,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.1136961430311203,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.16337041556835175,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.06925267726182938,
|
|
"rewards/len_reward/p75": 0.32950764894485474,
|
|
"rewards/len_reward/var": 0.07344760000705719,
|
|
"rewards/missing_response_penalty": -0.109375,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.09792213141918182,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 8,
|
|
"value_loss": 2.640625,
|
|
"values": -0.8079665899276733,
|
|
"values/max": 1.77435302734375,
|
|
"values/median": -0.7548828125,
|
|
"values/min": -1.957733154296875,
|
|
"values/p25": -0.7548828125,
|
|
"values/p75": -0.7548828125,
|
|
"values/var": 0.18137165904045105
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 6.243507232284173e-05,
|
|
"adam_stats/lr_effective_mean": -1.6910949030002342e-10,
|
|
"adam_stats/lr_effective_min": -6.287322321441025e-05,
|
|
"adam_stats/m_t_max": 0.007395743392407894,
|
|
"adam_stats/m_t_mean": -2.0997197036631832e-11,
|
|
"adam_stats/m_t_min": -0.004758418537676334,
|
|
"adam_stats/v_t_max": 2.132852932845708e-05,
|
|
"adam_stats/v_t_mean": 1.3262481390885483e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.20253164556962025,
|
|
"eval_accuracy_reward": 0.6266666650772095,
|
|
"eval_accuracy_reward/correct": 0.9999999403953552,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23473802208900452,
|
|
"eval_advantages": 0.6266666650772095,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23473802208900452,
|
|
"eval_all_logprobs": -0.026214059442281723,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -8.9375,
|
|
"eval_all_logprobs/p25": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.040528010576963425,
|
|
"eval_answer_logprob_reward": 2.9724133014678955,
|
|
"eval_answer_logprob_reward/correct": 3.1414499282836914,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.047927774488925934,
|
|
"eval_answer_logprob_reward/incorrect": 2.688673973083496,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.272965669631958,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.4752677083015442,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.272965669631958,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.25453484058380127,
|
|
"eval_completion_length": 592.52001953125,
|
|
"eval_completion_length/correct": 477.4201965332031,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 454.0,
|
|
"eval_completion_length/correct/min": 167.0,
|
|
"eval_completion_length/correct/p25": 318.75,
|
|
"eval_completion_length/correct/p75": 612.75,
|
|
"eval_completion_length/correct/var": 40107.1875,
|
|
"eval_completion_length/incorrect": 785.7232666015625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 880.0,
|
|
"eval_completion_length/incorrect/min": 167.0,
|
|
"eval_completion_length/incorrect/p25": 580.25,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 69330.3671875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 555.0,
|
|
"eval_completion_length/min": 167.0,
|
|
"eval_completion_length/p25": 357.0,
|
|
"eval_completion_length/p75": 806.0,
|
|
"eval_completion_length/var": 73133.8125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 1457583.0,
|
|
"eval_len_reward": 0.044376272708177567,
|
|
"eval_len_reward/correct": 0.17966805398464203,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.20304115116596222,
|
|
"eval_len_reward/correct/min": -0.489673912525177,
|
|
"eval_len_reward/correct/p25": 0.03819954767823219,
|
|
"eval_len_reward/correct/p75": 0.3427225351333618,
|
|
"eval_len_reward/correct/var": 0.04893714562058449,
|
|
"eval_len_reward/incorrect": -0.18272064626216888,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.15007656812667847,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.3179081678390503,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03250543028116226,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.13343378901481628,
|
|
"eval_len_reward/p75": 0.27335086464881897,
|
|
"eval_len_reward/var": 0.07350050657987595,
|
|
"eval_mean_logprobs": -0.025634765625,
|
|
"eval_mean_logprobs/var": 0.0003719329833984375,
|
|
"eval_missing_response_penalty": -0.1366666704416275,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.3660714328289032,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.23415379226207733,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.1183835044503212,
|
|
"eval_num_completions/total": 2400,
|
|
"eval_pred_value_log_probs": -0.662541925907135,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.70703125,
|
|
"eval_pred_value_log_probs/min": -1.0546875,
|
|
"eval_pred_value_log_probs/p25": -0.70703125,
|
|
"eval_pred_value_log_probs/p75": -0.70703125,
|
|
"eval_pred_value_log_probs/var": 0.02536529116332531,
|
|
"eval_reward": 0.6266666650772095,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23473802208900452,
|
|
"eval_rewards/accuracy_reward": 0.6266666650772095,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23473802208900452,
|
|
"eval_rewards/answer_logprob_reward": 2.9724133014678955,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.272965669631958,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.25453484058380127,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.044376272708177567,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.13343378901481628,
|
|
"eval_rewards/len_reward/p75": 0.27335086464881897,
|
|
"eval_rewards/len_reward/var": 0.07350050657987595,
|
|
"eval_rewards/missing_response_penalty": -0.1366666704416275,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.1183835044503212,
|
|
"eval_runtime": 171.5273,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8006720542907715,
|
|
"eval_values/max": 1.9754486083984375,
|
|
"eval_values/median": -0.76513671875,
|
|
"eval_values/min": -1.9915542602539062,
|
|
"eval_values/p25": -0.76513671875,
|
|
"eval_values/p75": -0.76513671875,
|
|
"eval_values/var": 0.1165805384516716,
|
|
"step": 8
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7916666865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.16579407453536987,
|
|
"adam_stats/lr_effective_max": 6.244866381166503e-05,
|
|
"adam_stats/lr_effective_mean": -1.4529581704447736e-10,
|
|
"adam_stats/lr_effective_min": -6.220162322279066e-05,
|
|
"adam_stats/m_t_max": 0.004721947014331818,
|
|
"adam_stats/m_t_mean": -2.0175234746733217e-11,
|
|
"adam_stats/m_t_min": -0.004131162539124489,
|
|
"adam_stats/v_t_max": 2.23145070776809e-05,
|
|
"adam_stats/v_t_mean": 1.3832478999026288e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7916666865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.16579407453536987,
|
|
"all_logprobs": -0.02305363304913044,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -9.125,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.03668288514018059,
|
|
"answer_logprob_reward": 3.02274751663208,
|
|
"answer_logprob_reward/correct": 3.061724901199341,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.08524507284164429,
|
|
"answer_logprob_reward/incorrect": 2.8746325969696045,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.18981382250785828,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 2.079441547393799,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.11195386201143265,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 533.984375,
|
|
"completion_length/correct": 463.02630615234375,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 413.0,
|
|
"completion_length/correct/min": 175.0,
|
|
"completion_length/correct/p25": 323.75,
|
|
"completion_length/correct/p75": 546.5,
|
|
"completion_length/correct/var": 37277.74609375,
|
|
"completion_length/incorrect": 803.625,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 804.0,
|
|
"completion_length/incorrect/min": 259.0,
|
|
"completion_length/incorrect/p25": 660.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 58065.87890625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 457.0,
|
|
"completion_length/min": 175.0,
|
|
"completion_length/p25": 344.0,
|
|
"completion_length/p75": 686.75,
|
|
"completion_length/var": 60560.6171875,
|
|
"epoch": 0.22784810126582278,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1036022.0,
|
|
"grad_norm": 0.36979013681411743,
|
|
"kl": 0.007781982421875,
|
|
"learning_rate": 9.504844339512096e-06,
|
|
"len_reward": 0.13034191727638245,
|
|
"len_reward/correct": 0.21805612742900848,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2795400619506836,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.10460343956947327,
|
|
"len_reward/correct/p75": 0.3683248460292816,
|
|
"len_reward/correct/var": 0.04483015090227127,
|
|
"len_reward/incorrect": -0.2029721736907959,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1942392885684967,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3398860991001129,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03722740337252617,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.1930946260690689,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.0009001440485008061,
|
|
"len_reward/p75": 0.3444806933403015,
|
|
"len_reward/var": 0.0724324956536293,
|
|
"loss": -0.7914,
|
|
"mean_logprobs": -0.022216796875,
|
|
"mean_logprobs/var": 0.00019550323486328125,
|
|
"missing_response_penalty": -0.0677083358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.32500001788139343,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.22500000894069672,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.06345440447330475,
|
|
"non_smoothed_logprobs/min": -19.9375,
|
|
"num_completions/total": 1728,
|
|
"policy_loss": -0.7913558483123779,
|
|
"policy_loss/max": 0.0004762290045619011,
|
|
"policy_loss/median": -0.9996488690376282,
|
|
"policy_loss/min": -0.9999149441719055,
|
|
"policy_loss/p25": -0.9997439980506897,
|
|
"policy_loss/p75": -0.9994585514068604,
|
|
"policy_loss/var": 0.16577255725860596,
|
|
"pred_value_log_probs": -0.6520935297012329,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.70703125,
|
|
"pred_value_log_probs/min": -1.140625,
|
|
"pred_value_log_probs/p25": -0.70703125,
|
|
"pred_value_log_probs/p75": -0.70703125,
|
|
"pred_value_log_probs/var": 0.041593849658966064,
|
|
"reward": 0.7916666865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.16579407453536987,
|
|
"rewards/accuracy_reward": 0.7916666865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.16579407453536987,
|
|
"rewards/answer_logprob_reward": 3.02274751663208,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 2.079441547393799,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.11195386201143265,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.13034191727638245,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.1930946260690689,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.0009001440485008061,
|
|
"rewards/len_reward/p75": 0.3444806933403015,
|
|
"rewards/len_reward/var": 0.0724324956536293,
|
|
"rewards/missing_response_penalty": -0.0677083358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.06345440447330475,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 9,
|
|
"value_loss": 2.953125,
|
|
"values": -0.8088086843490601,
|
|
"values/max": 1.8154296875,
|
|
"values/median": -0.76513671875,
|
|
"values/min": -2.000594139099121,
|
|
"values/p25": -0.76513671875,
|
|
"values/p75": -0.76513671875,
|
|
"values/var": 0.20579427480697632
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 6.244866381166503e-05,
|
|
"adam_stats/lr_effective_mean": -1.4529581704447736e-10,
|
|
"adam_stats/lr_effective_min": -6.220162322279066e-05,
|
|
"adam_stats/m_t_max": 0.004721947014331818,
|
|
"adam_stats/m_t_mean": -2.0175234746733217e-11,
|
|
"adam_stats/m_t_min": -0.004131162539124489,
|
|
"adam_stats/v_t_max": 2.23145070776809e-05,
|
|
"adam_stats/v_t_mean": 1.3832478999026288e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.22784810126582278,
|
|
"eval_accuracy_reward": 0.6700000166893005,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22183947265148163,
|
|
"eval_advantages": 0.6700000166893005,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22183947265148163,
|
|
"eval_all_logprobs": -0.023103248327970505,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -10.375,
|
|
"eval_all_logprobs/p25": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.03600381314754486,
|
|
"eval_answer_logprob_reward": 3.008866548538208,
|
|
"eval_answer_logprob_reward/correct": 3.1257662773132324,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.0561484731733799,
|
|
"eval_answer_logprob_reward/incorrect": 2.7715237140655518,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.4142383635044098,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.20116610825061798,
|
|
"eval_completion_length": 577.9866943359375,
|
|
"eval_completion_length/correct": 494.14923095703125,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 442.0,
|
|
"eval_completion_length/correct/min": 155.0,
|
|
"eval_completion_length/correct/p25": 323.0,
|
|
"eval_completion_length/correct/p75": 641.0,
|
|
"eval_completion_length/correct/var": 47330.5234375,
|
|
"eval_completion_length/incorrect": 748.2020263671875,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 759.0,
|
|
"eval_completion_length/incorrect/min": 187.0,
|
|
"eval_completion_length/incorrect/p25": 563.5,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 66543.4140625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 525.0,
|
|
"eval_completion_length/min": 155.0,
|
|
"eval_completion_length/p25": 365.75,
|
|
"eval_completion_length/p75": 761.25,
|
|
"eval_completion_length/var": 67787.5703125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 1630979.0,
|
|
"eval_len_reward": 0.043535925447940826,
|
|
"eval_len_reward/correct": 0.14882901310920715,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1847875714302063,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.005553270224481821,
|
|
"eval_len_reward/correct/p75": 0.32223182916641235,
|
|
"eval_len_reward/correct/var": 0.058313969522714615,
|
|
"eval_len_reward/incorrect": -0.17024099826812744,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.13499315083026886,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.30066239833831787,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.030154937878251076,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.005553270224481821,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.14721713960170746,
|
|
"eval_len_reward/p75": 0.24546536803245544,
|
|
"eval_len_reward/var": 0.07147407531738281,
|
|
"eval_mean_logprobs": -0.0224609375,
|
|
"eval_mean_logprobs/var": 0.0003147125244140625,
|
|
"eval_missing_response_penalty": -0.11666666716337204,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.35353535413742065,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.2308802306652069,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.10340021550655365,
|
|
"eval_num_completions/total": 2700,
|
|
"eval_pred_value_log_probs": -0.6526988744735718,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.69921875,
|
|
"eval_pred_value_log_probs/min": -0.9375,
|
|
"eval_pred_value_log_probs/p25": -0.69921875,
|
|
"eval_pred_value_log_probs/p75": -0.69921875,
|
|
"eval_pred_value_log_probs/var": 0.026807580143213272,
|
|
"eval_reward": 0.6700000166893005,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22183947265148163,
|
|
"eval_rewards/accuracy_reward": 0.6700000166893005,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22183947265148163,
|
|
"eval_rewards/answer_logprob_reward": 3.008866548538208,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.20116610825061798,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.043535925447940826,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.005553270224481821,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.14721713960170746,
|
|
"eval_rewards/len_reward/p75": 0.24546536803245544,
|
|
"eval_rewards/len_reward/var": 0.07147407531738281,
|
|
"eval_rewards/missing_response_penalty": -0.11666666716337204,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.10340021550655365,
|
|
"eval_runtime": 172.2124,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.812279999256134,
|
|
"eval_values/max": 1.9834823608398438,
|
|
"eval_values/median": -0.77294921875,
|
|
"eval_values/min": -2.0000133514404297,
|
|
"eval_values/p25": -0.77294921875,
|
|
"eval_values/p75": -0.77294921875,
|
|
"eval_values/var": 0.12297380715608597,
|
|
"step": 9
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7760416865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.1747109591960907,
|
|
"adam_stats/lr_effective_max": 6.146151281427592e-05,
|
|
"adam_stats/lr_effective_mean": -6.28145313541495e-11,
|
|
"adam_stats/lr_effective_min": -6.176823808345944e-05,
|
|
"adam_stats/m_t_max": 0.0044099693186581135,
|
|
"adam_stats/m_t_mean": -3.2349935230602256e-11,
|
|
"adam_stats/m_t_min": -0.003992704674601555,
|
|
"adam_stats/v_t_max": 2.233444683952257e-05,
|
|
"adam_stats/v_t_mean": 1.3899149843218939e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7760416865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.1747109591960907,
|
|
"all_logprobs": -0.017746614292263985,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -8.9375,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.02617828920483589,
|
|
"answer_logprob_reward": 2.9540839195251465,
|
|
"answer_logprob_reward/correct": 3.0880846977233887,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06667309999465942,
|
|
"answer_logprob_reward/incorrect": 2.4897546768188477,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/min": 1.3862943649291992,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.43310627341270447,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.3862943649291992,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.20944732427597046,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 571.4896240234375,
|
|
"completion_length/correct": 498.16778564453125,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 453.0,
|
|
"completion_length/correct/min": 160.0,
|
|
"completion_length/correct/p25": 347.0,
|
|
"completion_length/correct/p75": 618.0,
|
|
"completion_length/correct/var": 42527.9140625,
|
|
"completion_length/incorrect": 825.5581665039062,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 289.0,
|
|
"completion_length/incorrect/p25": 640.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 60186.01953125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 501.0,
|
|
"completion_length/min": 160.0,
|
|
"completion_length/p25": 381.25,
|
|
"completion_length/p75": 755.0,
|
|
"completion_length/var": 64914.48046875,
|
|
"epoch": 0.25316455696202533,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1145748.0,
|
|
"grad_norm": 0.1811712235212326,
|
|
"kl": 0.00775146484375,
|
|
"learning_rate": 9.292243968009332e-06,
|
|
"len_reward": 0.09788952767848969,
|
|
"len_reward/correct": 0.18229344487190247,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2094140201807022,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.07336561381816864,
|
|
"len_reward/correct/p75": 0.3368038833141327,
|
|
"len_reward/correct/var": 0.04961439594626427,
|
|
"len_reward/incorrect": -0.19457988440990448,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.16489103436470032,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.344448059797287,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03579093515872955,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.1380111575126648,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.0582333579659462,
|
|
"len_reward/p75": 0.2976534962654114,
|
|
"len_reward/var": 0.07112973183393478,
|
|
"loss": -0.7757,
|
|
"mean_logprobs": -0.017333984375,
|
|
"mean_logprobs/var": 0.00017452239990234375,
|
|
"missing_response_penalty": -0.1145833358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.5116279125213623,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.25581395626068115,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.10198517143726349,
|
|
"non_smoothed_logprobs/min": -20.1875,
|
|
"num_completions/total": 1920,
|
|
"policy_loss": -0.7757319808006287,
|
|
"policy_loss/max": 0.000476124812848866,
|
|
"policy_loss/median": -0.9996414184570312,
|
|
"policy_loss/min": -0.9998821020126343,
|
|
"policy_loss/p25": -0.9997469186782837,
|
|
"policy_loss/p75": -0.9994195103645325,
|
|
"policy_loss/var": 0.17468826472759247,
|
|
"pred_value_log_probs": -0.6530084609985352,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.69921875,
|
|
"pred_value_log_probs/min": -1.28125,
|
|
"pred_value_log_probs/p25": -0.69921875,
|
|
"pred_value_log_probs/p75": -0.69921875,
|
|
"pred_value_log_probs/var": 0.0391046479344368,
|
|
"reward": 0.7760416865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.1747109591960907,
|
|
"rewards/accuracy_reward": 0.7760416865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.1747109591960907,
|
|
"rewards/answer_logprob_reward": 2.9540839195251465,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.3862943649291992,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.20944732427597046,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09788952767848969,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.1380111575126648,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.0582333579659462,
|
|
"rewards/len_reward/p75": 0.2976534962654114,
|
|
"rewards/len_reward/var": 0.07112973183393478,
|
|
"rewards/missing_response_penalty": -0.1145833358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.10198517143726349,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 10,
|
|
"value_loss": 2.8359375,
|
|
"values": -0.7617847323417664,
|
|
"values/max": 2.0,
|
|
"values/median": -0.77294921875,
|
|
"values/min": -1.9896049499511719,
|
|
"values/p25": -0.77294921875,
|
|
"values/p75": -0.77294921875,
|
|
"values/var": 0.22872459888458252
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 6.146151281427592e-05,
|
|
"adam_stats/lr_effective_mean": -6.28145313541495e-11,
|
|
"adam_stats/lr_effective_min": -6.176823808345944e-05,
|
|
"adam_stats/m_t_max": 0.0044099693186581135,
|
|
"adam_stats/m_t_mean": -3.2349935230602256e-11,
|
|
"adam_stats/m_t_min": -0.003992704674601555,
|
|
"adam_stats/v_t_max": 2.233444683952257e-05,
|
|
"adam_stats/v_t_mean": 1.3899149843218939e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.25316455696202533,
|
|
"eval_accuracy_reward": 0.6200000047683716,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2363879680633545,
|
|
"eval_advantages": 0.6200000047683716,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2363879680633545,
|
|
"eval_all_logprobs": -0.017270084470510483,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.53125,
|
|
"eval_all_logprobs/p25": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02553660236299038,
|
|
"eval_answer_logprob_reward": 3.0562992095947266,
|
|
"eval_answer_logprob_reward/correct": 3.1368906497955322,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.05037706345319748,
|
|
"eval_answer_logprob_reward/incorrect": 2.9248080253601074,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.2635025382041931,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.14138683676719666,
|
|
"eval_completion_length": 558.9967041015625,
|
|
"eval_completion_length/correct": 473.81182861328125,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 434.0,
|
|
"eval_completion_length/correct/min": 104.0,
|
|
"eval_completion_length/correct/p25": 319.75,
|
|
"eval_completion_length/correct/p75": 600.25,
|
|
"eval_completion_length/correct/var": 41081.31640625,
|
|
"eval_completion_length/incorrect": 697.9824829101562,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 662.0,
|
|
"eval_completion_length/incorrect/min": 167.0,
|
|
"eval_completion_length/incorrect/p25": 442.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 74562.5859375,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 490.0,
|
|
"eval_completion_length/min": 104.0,
|
|
"eval_completion_length/p25": 351.75,
|
|
"eval_completion_length/p75": 730.25,
|
|
"eval_completion_length/var": 65476.453125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 1798678.0,
|
|
"eval_len_reward": 0.047263771295547485,
|
|
"eval_len_reward/correct": 0.1697932779788971,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.17877094447612762,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.04317869991064072,
|
|
"eval_len_reward/correct/p75": 0.3401416540145874,
|
|
"eval_len_reward/correct/var": 0.0487174354493618,
|
|
"eval_len_reward/incorrect": -0.1526528149843216,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.07299452275037766,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2952413558959961,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03232886269688606,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.08298538625240326,
|
|
"eval_len_reward/p75": 0.24518707394599915,
|
|
"eval_len_reward/var": 0.06693842262029648,
|
|
"eval_mean_logprobs": -0.0174560546875,
|
|
"eval_mean_logprobs/var": 0.0001621246337890625,
|
|
"eval_missing_response_penalty": -0.10000000149011612,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2631579041481018,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.1956217885017395,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.09030100703239441,
|
|
"eval_num_completions/total": 3000,
|
|
"eval_pred_value_log_probs": -0.6529466509819031,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.703125,
|
|
"eval_pred_value_log_probs/min": -1.0234375,
|
|
"eval_pred_value_log_probs/p25": -0.703125,
|
|
"eval_pred_value_log_probs/p75": -0.703125,
|
|
"eval_pred_value_log_probs/var": 0.028898609802126884,
|
|
"eval_reward": 0.6200000047683716,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2363879680633545,
|
|
"eval_rewards/accuracy_reward": 0.6200000047683716,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2363879680633545,
|
|
"eval_rewards/answer_logprob_reward": 3.0562992095947266,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.14138683676719666,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.047263771295547485,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.08298538625240326,
|
|
"eval_rewards/len_reward/p75": 0.24518707394599915,
|
|
"eval_rewards/len_reward/var": 0.06693842262029648,
|
|
"eval_rewards/missing_response_penalty": -0.10000000149011612,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.09030100703239441,
|
|
"eval_runtime": 169.9197,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8177988529205322,
|
|
"eval_values/max": 1.9912681579589844,
|
|
"eval_values/median": -0.77294921875,
|
|
"eval_values/min": -2.000148296356201,
|
|
"eval_values/p25": -0.77294921875,
|
|
"eval_values/p75": -0.77294921875,
|
|
"eval_values/var": 0.13172577321529388,
|
|
"step": 10
|
|
},
|
|
{
|
|
"accuracy_reward": 0.765625,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.1803828477859497,
|
|
"adam_stats/lr_effective_max": 6.084773485781625e-05,
|
|
"adam_stats/lr_effective_mean": 3.8493843529385785e-11,
|
|
"adam_stats/lr_effective_min": -6.085845598136075e-05,
|
|
"adam_stats/m_t_max": 0.004168862476944923,
|
|
"adam_stats/m_t_mean": -3.178726379338137e-11,
|
|
"adam_stats/m_t_min": -0.003922298084944487,
|
|
"adam_stats/v_t_max": 2.2330892534228042e-05,
|
|
"adam_stats/v_t_mean": 1.4039654855357853e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.765625,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.1803828477859497,
|
|
"all_logprobs": -0.018302688375115395,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.59375,
|
|
"all_logprobs/p25": -2.384185791015625e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.027053385972976685,
|
|
"answer_logprob_reward": 2.915700912475586,
|
|
"answer_logprob_reward/correct": 3.0188896656036377,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.09472610056400299,
|
|
"answer_logprob_reward/incorrect": 2.578617811203003,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.2321436405181885,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.5363913178443909,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.2321436405181885,
|
|
"answer_logprob_reward/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.23094026744365692,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 548.84375,
|
|
"completion_length/correct": 481.9727783203125,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 452.0,
|
|
"completion_length/correct/min": 141.0,
|
|
"completion_length/correct/p25": 315.0,
|
|
"completion_length/correct/p75": 613.5,
|
|
"completion_length/correct/var": 43214.8671875,
|
|
"completion_length/incorrect": 767.2888793945312,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 864.0,
|
|
"completion_length/incorrect/min": 261.0,
|
|
"completion_length/incorrect/p25": 560.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 74908.5703125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 490.0,
|
|
"completion_length/min": 141.0,
|
|
"completion_length/p25": 334.0,
|
|
"completion_length/p75": 717.25,
|
|
"completion_length/var": 64973.890625,
|
|
"epoch": 0.27848101265822783,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1251126.0,
|
|
"grad_norm": 0.23812517523765564,
|
|
"kl": 0.0096435546875,
|
|
"learning_rate": 9.045084971874738e-06,
|
|
"len_reward": 0.056315742433071136,
|
|
"len_reward/correct": 0.1266818791627884,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.20374064147472382,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.09587077796459198,
|
|
"len_reward/correct/p75": 0.3539902865886688,
|
|
"len_reward/correct/var": 0.07949673384428024,
|
|
"len_reward/incorrect": -0.1735469400882721,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.08390804380178452,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.34236451983451843,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.037365350872278214,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.0015723269898444414,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.14850373566150665,
|
|
"len_reward/p75": 0.31304460763931274,
|
|
"len_reward/var": 0.08563410490751266,
|
|
"loss": -0.7652,
|
|
"mean_logprobs": -0.01708984375,
|
|
"mean_logprobs/var": 0.00019168853759765625,
|
|
"missing_response_penalty": -0.09375,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4000000059604645,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2454545497894287,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.08540575951337814,
|
|
"non_smoothed_logprobs/min": -19.0,
|
|
"num_completions/total": 2112,
|
|
"policy_loss": -0.7652397155761719,
|
|
"policy_loss/max": 0.0006840777932666242,
|
|
"policy_loss/median": -0.999577522277832,
|
|
"policy_loss/min": -0.9998697638511658,
|
|
"policy_loss/p25": -0.999678373336792,
|
|
"policy_loss/p75": -0.9991584420204163,
|
|
"policy_loss/var": 0.18036982417106628,
|
|
"pred_value_log_probs": -0.6432632207870483,
|
|
"pred_value_log_probs/max": -0.0003204345703125,
|
|
"pred_value_log_probs/median": -0.703125,
|
|
"pred_value_log_probs/min": -1.3515625,
|
|
"pred_value_log_probs/p25": -0.703125,
|
|
"pred_value_log_probs/p75": -0.703125,
|
|
"pred_value_log_probs/var": 0.03457346186041832,
|
|
"reward": 0.765625,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.1803828477859497,
|
|
"rewards/accuracy_reward": 0.765625,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.1803828477859497,
|
|
"rewards/answer_logprob_reward": 2.915700912475586,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.2321436405181885,
|
|
"rewards/answer_logprob_reward/p25": 2.4849066734313965,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.23094026744365692,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.056315742433071136,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.0015723269898444414,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.14850373566150665,
|
|
"rewards/len_reward/p75": 0.31304460763931274,
|
|
"rewards/len_reward/var": 0.08563410490751266,
|
|
"rewards/missing_response_penalty": -0.09375,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.08540575951337814,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 11,
|
|
"value_loss": 2.7578125,
|
|
"values": -0.8260021209716797,
|
|
"values/max": 1.342041015625,
|
|
"values/median": -0.77294921875,
|
|
"values/min": -1.9998247623443604,
|
|
"values/p25": -0.77294921875,
|
|
"values/p75": -0.77294921875,
|
|
"values/var": 0.20274372398853302
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 6.084773485781625e-05,
|
|
"adam_stats/lr_effective_mean": 3.8493843529385785e-11,
|
|
"adam_stats/lr_effective_min": -6.085845598136075e-05,
|
|
"adam_stats/m_t_max": 0.004168862476944923,
|
|
"adam_stats/m_t_mean": -3.178726379338137e-11,
|
|
"adam_stats/m_t_min": -0.003922298084944487,
|
|
"adam_stats/v_t_max": 2.2330892534228042e-05,
|
|
"adam_stats/v_t_mean": 1.4039654855357853e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.27848101265822783,
|
|
"eval_accuracy_reward": 0.6600000262260437,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22515049576759338,
|
|
"eval_advantages": 0.6600000262260437,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22515049576759338,
|
|
"eval_all_logprobs": -0.015983078628778458,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.0,
|
|
"eval_all_logprobs/p25": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.024440567940473557,
|
|
"eval_answer_logprob_reward": 3.0050878524780273,
|
|
"eval_answer_logprob_reward/correct": 3.1009719371795654,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.08625704050064087,
|
|
"eval_answer_logprob_reward/incorrect": 2.8189597129821777,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.36717504262924194,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1987670212984085,
|
|
"eval_completion_length": 563.7766723632812,
|
|
"eval_completion_length/correct": 480.99493408203125,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 439.0,
|
|
"eval_completion_length/correct/min": 103.0,
|
|
"eval_completion_length/correct/p25": 319.0,
|
|
"eval_completion_length/correct/p75": 629.0,
|
|
"eval_completion_length/correct/var": 41770.5625,
|
|
"eval_completion_length/incorrect": 724.4706420898438,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 722.0,
|
|
"eval_completion_length/incorrect/min": 203.0,
|
|
"eval_completion_length/incorrect/p25": 480.25,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 77053.21875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 508.0,
|
|
"eval_completion_length/min": 103.0,
|
|
"eval_completion_length/p25": 351.25,
|
|
"eval_completion_length/p75": 733.25,
|
|
"eval_completion_length/var": 66896.09375,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 1967811.0,
|
|
"eval_len_reward": 0.057859741151332855,
|
|
"eval_len_reward/correct": 0.16566860675811768,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.18490415811538696,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.004859698936343193,
|
|
"eval_len_reward/correct/p75": 0.3304007947444916,
|
|
"eval_len_reward/correct/var": 0.04873096942901611,
|
|
"eval_len_reward/incorrect": -0.15141631662845612,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.0882587879896164,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.27317923307418823,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.031230084598064423,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.09351489692926407,
|
|
"eval_len_reward/p75": 0.25090155005455017,
|
|
"eval_len_reward/var": 0.06529360264539719,
|
|
"eval_mean_logprobs": -0.01611328125,
|
|
"eval_mean_logprobs/var": 0.00013828277587890625,
|
|
"eval_missing_response_penalty": -0.10999999940395355,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.3235294222831726,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.2210250347852707,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.09822742640972137,
|
|
"eval_num_completions/total": 3300,
|
|
"eval_pred_value_log_probs": -0.6458382606506348,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6953125,
|
|
"eval_pred_value_log_probs/min": -1.0703125,
|
|
"eval_pred_value_log_probs/p25": -0.6953125,
|
|
"eval_pred_value_log_probs/p75": -0.6953125,
|
|
"eval_pred_value_log_probs/var": 0.028838587924838066,
|
|
"eval_reward": 0.6600000262260437,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22515049576759338,
|
|
"eval_rewards/accuracy_reward": 0.6600000262260437,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22515049576759338,
|
|
"eval_rewards/answer_logprob_reward": 3.0050878524780273,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1987670212984085,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.057859741151332855,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.09351489692926407,
|
|
"eval_rewards/len_reward/p75": 0.25090155005455017,
|
|
"eval_rewards/len_reward/var": 0.06529360264539719,
|
|
"eval_rewards/missing_response_penalty": -0.10999999940395355,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.09822742640972137,
|
|
"eval_runtime": 165.0613,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8255552649497986,
|
|
"eval_values/max": 1.9914283752441406,
|
|
"eval_values/median": -0.78125,
|
|
"eval_values/min": -2.0001308917999268,
|
|
"eval_values/p25": -0.78125,
|
|
"eval_values/p75": -0.78125,
|
|
"eval_values/var": 0.13591179251670837,
|
|
"step": 11
|
|
},
|
|
{
|
|
"accuracy_reward": 0.8020833730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.15957680344581604,
|
|
"adam_stats/lr_effective_max": 5.870972745469771e-05,
|
|
"adam_stats/lr_effective_mean": -8.81928488571937e-11,
|
|
"adam_stats/lr_effective_min": -5.807427078252658e-05,
|
|
"adam_stats/m_t_max": 0.0037859270814806223,
|
|
"adam_stats/m_t_mean": -7.138125594080558e-12,
|
|
"adam_stats/m_t_min": -0.0038272289093583822,
|
|
"adam_stats/v_t_max": 2.269002652610652e-05,
|
|
"adam_stats/v_t_mean": 1.4284688883148267e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.8020833730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.15957680344581604,
|
|
"all_logprobs": -0.014600125141441822,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.75,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.020908931270241737,
|
|
"answer_logprob_reward": 3.0444083213806152,
|
|
"answer_logprob_reward/correct": 3.0883731842041016,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06903017312288284,
|
|
"answer_logprob_reward/incorrect": 2.8662333488464355,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.2063932716846466,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 2.079441547393799,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.10315284132957458,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 543.3385620117188,
|
|
"completion_length/correct": 488.8571472167969,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 474.0,
|
|
"completion_length/correct/min": 160.0,
|
|
"completion_length/correct/p25": 329.25,
|
|
"completion_length/correct/p75": 601.0,
|
|
"completion_length/correct/var": 33187.94140625,
|
|
"completion_length/incorrect": 764.131591796875,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 741.0,
|
|
"completion_length/incorrect/min": 243.0,
|
|
"completion_length/incorrect/p25": 543.25,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 62576.92578125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 504.0,
|
|
"completion_length/min": 160.0,
|
|
"completion_length/p25": 372.75,
|
|
"completion_length/p75": 662.0,
|
|
"completion_length/var": 50799.4296875,
|
|
"epoch": 0.3037974683544304,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1355447.0,
|
|
"grad_norm": 0.2768768072128296,
|
|
"kl": 0.010009765625,
|
|
"learning_rate": 8.765357330018056e-06,
|
|
"len_reward": 0.07785402238368988,
|
|
"len_reward/correct": 0.14999371767044067,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.17182274162769318,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.0,
|
|
"len_reward/correct/p75": 0.3500424921512604,
|
|
"len_reward/correct/var": 0.06163345277309418,
|
|
"len_reward/incorrect": -0.21450167894363403,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.20623500645160675,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3933675289154053,
|
|
"len_reward/incorrect/p75": -0.006477926857769489,
|
|
"len_reward/incorrect/var": 0.037543874233961105,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.07889688014984131,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.08291997760534286,
|
|
"len_reward/p75": 0.327726274728775,
|
|
"len_reward/var": 0.07784508168697357,
|
|
"loss": -0.8017,
|
|
"mean_logprobs": -0.014404296875,
|
|
"mean_logprobs/var": 0.00012302398681640625,
|
|
"missing_response_penalty": -0.0729166716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.3684210479259491,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.23897583782672882,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.06795375049114227,
|
|
"non_smoothed_logprobs/min": -18.625,
|
|
"num_completions/total": 2304,
|
|
"policy_loss": -0.8016828298568726,
|
|
"policy_loss/max": 0.0008261256152763963,
|
|
"policy_loss/median": -0.9995740056037903,
|
|
"policy_loss/min": -0.9999816417694092,
|
|
"policy_loss/p25": -0.9996675252914429,
|
|
"policy_loss/p75": -0.9993911981582642,
|
|
"policy_loss/var": 0.1595751941204071,
|
|
"pred_value_log_probs": -0.6363335251808167,
|
|
"pred_value_log_probs/max": -0.0003261566162109375,
|
|
"pred_value_log_probs/median": -0.6953125,
|
|
"pred_value_log_probs/min": -1.109375,
|
|
"pred_value_log_probs/p25": -0.6953125,
|
|
"pred_value_log_probs/p75": -0.6953125,
|
|
"pred_value_log_probs/var": 0.03291379660367966,
|
|
"reward": 0.8020833730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.15957680344581604,
|
|
"rewards/accuracy_reward": 0.8020833730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.15957680344581604,
|
|
"rewards/answer_logprob_reward": 3.0444083213806152,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 2.079441547393799,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.10315284132957458,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.07785402238368988,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.07889688014984131,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.08291997760534286,
|
|
"rewards/len_reward/p75": 0.327726274728775,
|
|
"rewards/len_reward/var": 0.07784508168697357,
|
|
"rewards/missing_response_penalty": -0.0729166716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.06795375049114227,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 12,
|
|
"value_loss": 2.765625,
|
|
"values": -0.7823073267936707,
|
|
"values/max": 1.7805557250976562,
|
|
"values/median": -0.78125,
|
|
"values/min": -1.9998421669006348,
|
|
"values/p25": -0.78125,
|
|
"values/p75": -0.78125,
|
|
"values/var": 0.24507829546928406
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 5.870972745469771e-05,
|
|
"adam_stats/lr_effective_mean": -8.81928488571937e-11,
|
|
"adam_stats/lr_effective_min": -5.807427078252658e-05,
|
|
"adam_stats/m_t_max": 0.0037859270814806223,
|
|
"adam_stats/m_t_mean": -7.138125594080558e-12,
|
|
"adam_stats/m_t_min": -0.0038272289093583822,
|
|
"adam_stats/v_t_max": 2.269002652610652e-05,
|
|
"adam_stats/v_t_mean": 1.4284688883148267e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.3037974683544304,
|
|
"eval_accuracy_reward": 0.6600000262260437,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22515052556991577,
|
|
"eval_advantages": 0.6600000262260437,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22515052556991577,
|
|
"eval_all_logprobs": -0.01577628403902054,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -10.125,
|
|
"eval_all_logprobs/p25": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.024152709171175957,
|
|
"eval_answer_logprob_reward": 3.056529998779297,
|
|
"eval_answer_logprob_reward/correct": 3.141859292984009,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.047693025320768356,
|
|
"eval_answer_logprob_reward/incorrect": 2.890890598297119,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.27481406927108765,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.13843455910682678,
|
|
"eval_completion_length": 568.433349609375,
|
|
"eval_completion_length/correct": 497.3737487792969,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 461.0,
|
|
"eval_completion_length/correct/min": 115.0,
|
|
"eval_completion_length/correct/p25": 322.75,
|
|
"eval_completion_length/correct/p75": 633.5,
|
|
"eval_completion_length/correct/var": 48618.09765625,
|
|
"eval_completion_length/incorrect": 706.37255859375,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 716.0,
|
|
"eval_completion_length/incorrect/min": 172.0,
|
|
"eval_completion_length/incorrect/p25": 452.25,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 82445.921875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 524.0,
|
|
"eval_completion_length/min": 115.0,
|
|
"eval_completion_length/p25": 346.5,
|
|
"eval_completion_length/p75": 750.75,
|
|
"eval_completion_length/var": 69716.96875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 2138341.0,
|
|
"eval_len_reward": 0.030194932594895363,
|
|
"eval_len_reward/correct": 0.13411900401115417,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.13924558460712433,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.016798434779047966,
|
|
"eval_len_reward/correct/p75": 0.3000715374946594,
|
|
"eval_len_reward/correct/var": 0.05324889346957207,
|
|
"eval_len_reward/incorrect": -0.1715400218963623,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.11954765766859055,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.3578011691570282,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03252788633108139,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.12422685325145721,
|
|
"eval_len_reward/p75": 0.23109574615955353,
|
|
"eval_len_reward/var": 0.06710663437843323,
|
|
"eval_mean_logprobs": -0.0150146484375,
|
|
"eval_mean_logprobs/var": 0.00012302398681640625,
|
|
"eval_missing_response_penalty": -0.09666667133569717,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.28431373834609985,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.20549407601356506,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0876142606139183,
|
|
"eval_num_completions/total": 3600,
|
|
"eval_pred_value_log_probs": -0.6469375491142273,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6953125,
|
|
"eval_pred_value_log_probs/min": -1.078125,
|
|
"eval_pred_value_log_probs/p25": -0.6953125,
|
|
"eval_pred_value_log_probs/p75": -0.6953125,
|
|
"eval_pred_value_log_probs/var": 0.028731336817145348,
|
|
"eval_reward": 0.6600000262260437,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22515052556991577,
|
|
"eval_rewards/accuracy_reward": 0.6600000262260437,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22515052556991577,
|
|
"eval_rewards/answer_logprob_reward": 3.056529998779297,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.13843455910682678,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.030194932594895363,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.12422685325145721,
|
|
"eval_rewards/len_reward/p75": 0.23109574615955353,
|
|
"eval_rewards/len_reward/var": 0.06710663437843323,
|
|
"eval_rewards/missing_response_penalty": -0.09666667133569717,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0876142606139183,
|
|
"eval_runtime": 175.9355,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8294759392738342,
|
|
"eval_values/max": 1.9914474487304688,
|
|
"eval_values/median": -0.78515625,
|
|
"eval_values/min": -2.000096082687378,
|
|
"eval_values/p25": -0.78515625,
|
|
"eval_values/p75": -0.78515625,
|
|
"eval_values/var": 0.13602778315544128,
|
|
"step": 12
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7552083730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.18583661317825317,
|
|
"adam_stats/lr_effective_max": 5.697783126379363e-05,
|
|
"adam_stats/lr_effective_mean": -2.277365795944064e-10,
|
|
"adam_stats/lr_effective_min": -5.703235729015432e-05,
|
|
"adam_stats/m_t_max": 0.0037888039369136095,
|
|
"adam_stats/m_t_mean": -8.236701251607137e-12,
|
|
"adam_stats/m_t_min": -0.003288755426183343,
|
|
"adam_stats/v_t_max": 2.2761514628655277e-05,
|
|
"adam_stats/v_t_mean": 1.4496625471219216e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7552083730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.18583661317825317,
|
|
"all_logprobs": -0.013648364692926407,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.5625,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.021080132573843002,
|
|
"answer_logprob_reward": 2.9505443572998047,
|
|
"answer_logprob_reward/correct": 3.039424419403076,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 1.7917594909667969,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.10409815609455109,
|
|
"answer_logprob_reward/incorrect": 2.676339864730835,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.3862943649291992,
|
|
"answer_logprob_reward/incorrect/p25": 2.2821741104125977,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.41298890113830566,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.3862943649291992,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.20244458317756653,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 550.9739990234375,
|
|
"completion_length/correct": 476.0689697265625,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 440.0,
|
|
"completion_length/correct/min": 156.0,
|
|
"completion_length/correct/p25": 317.0,
|
|
"completion_length/correct/p75": 609.0,
|
|
"completion_length/correct/var": 40123.453125,
|
|
"completion_length/incorrect": 782.0637817382812,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 906.0,
|
|
"completion_length/incorrect/min": 238.0,
|
|
"completion_length/incorrect/p25": 510.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 74556.109375,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 485.0,
|
|
"completion_length/min": 156.0,
|
|
"completion_length/p25": 344.25,
|
|
"completion_length/p75": 719.5,
|
|
"completion_length/var": 65606.4765625,
|
|
"epoch": 0.3291139240506329,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1461234.0,
|
|
"grad_norm": 0.2585687041282654,
|
|
"kl": 0.01025390625,
|
|
"learning_rate": 8.455313244934324e-06,
|
|
"len_reward": 0.11482175439596176,
|
|
"len_reward/correct": 0.20802628993988037,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.22021225094795227,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.11475150287151337,
|
|
"len_reward/correct/p75": 0.3654124438762665,
|
|
"len_reward/correct/var": 0.042087990790605545,
|
|
"len_reward/incorrect": -0.172724187374115,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1230769231915474,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3250735104084015,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03574460372328758,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.17319704592227936,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": 0.0,
|
|
"len_reward/p75": 0.2954641878604889,
|
|
"len_reward/var": 0.06728082150220871,
|
|
"loss": -0.7548,
|
|
"mean_logprobs": -0.0135498046875,
|
|
"mean_logprobs/var": 8.96453857421875e-05,
|
|
"missing_response_penalty": -0.1041666716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.42553189396858215,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.24976873397827148,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.09380454570055008,
|
|
"non_smoothed_logprobs/min": -18.875,
|
|
"num_completions/total": 2496,
|
|
"policy_loss": -0.7547990679740906,
|
|
"policy_loss/max": 0.0008481450495310128,
|
|
"policy_loss/median": -0.9995503425598145,
|
|
"policy_loss/min": -0.9998537302017212,
|
|
"policy_loss/p25": -0.9996625185012817,
|
|
"policy_loss/p75": -0.9991025924682617,
|
|
"policy_loss/var": 0.18582956492900848,
|
|
"pred_value_log_probs": -0.6478489637374878,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6953125,
|
|
"pred_value_log_probs/min": -1.3671875,
|
|
"pred_value_log_probs/p25": -0.6953125,
|
|
"pred_value_log_probs/p75": -0.6953125,
|
|
"pred_value_log_probs/var": 0.06093765050172806,
|
|
"reward": 0.7552083730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.18583661317825317,
|
|
"rewards/accuracy_reward": 0.7552083730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.18583661317825317,
|
|
"rewards/answer_logprob_reward": 2.9505443572998047,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.3862943649291992,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.20244458317756653,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.11482175439596176,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.17319704592227936,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": 0.0,
|
|
"rewards/len_reward/p75": 0.2954641878604889,
|
|
"rewards/len_reward/var": 0.06728082150220871,
|
|
"rewards/missing_response_penalty": -0.1041666716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.09380454570055008,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 13,
|
|
"value_loss": 3.1015625,
|
|
"values": -0.8192496299743652,
|
|
"values/max": 1.9883270263671875,
|
|
"values/median": -0.78515625,
|
|
"values/min": -2.0010595321655273,
|
|
"values/p25": -0.78515625,
|
|
"values/p75": -0.78515625,
|
|
"values/var": 0.2388850301504135
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 5.697783126379363e-05,
|
|
"adam_stats/lr_effective_mean": -2.277365795944064e-10,
|
|
"adam_stats/lr_effective_min": -5.703235729015432e-05,
|
|
"adam_stats/m_t_max": 0.0037888039369136095,
|
|
"adam_stats/m_t_mean": -8.236701251607137e-12,
|
|
"adam_stats/m_t_min": -0.003288755426183343,
|
|
"adam_stats/v_t_max": 2.2761514628655277e-05,
|
|
"adam_stats/v_t_mean": 1.4496625471219216e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.3291139240506329,
|
|
"eval_accuracy_reward": 0.6333333253860474,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23299884796142578,
|
|
"eval_advantages": 0.6333333253860474,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23299884796142578,
|
|
"eval_all_logprobs": -0.015315653756260872,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -9.875,
|
|
"eval_all_logprobs/p25": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.024531463161110878,
|
|
"eval_answer_logprob_reward": 3.0196218490600586,
|
|
"eval_answer_logprob_reward/correct": 3.109705924987793,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07587447762489319,
|
|
"eval_answer_logprob_reward/incorrect": 2.8640220165252686,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.32031023502349854,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1787933111190796,
|
|
"eval_completion_length": 557.0900268554688,
|
|
"eval_completion_length/correct": 483.5684509277344,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 453.0,
|
|
"eval_completion_length/correct/min": 121.0,
|
|
"eval_completion_length/correct/p25": 328.0,
|
|
"eval_completion_length/correct/p75": 615.0,
|
|
"eval_completion_length/correct/var": 44640.59765625,
|
|
"eval_completion_length/incorrect": 684.081787109375,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 665.0,
|
|
"eval_completion_length/incorrect/min": 192.0,
|
|
"eval_completion_length/incorrect/p25": 433.5,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 75831.328125,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 512.0,
|
|
"eval_completion_length/min": 121.0,
|
|
"eval_completion_length/p25": 345.75,
|
|
"eval_completion_length/p75": 743.0,
|
|
"eval_completion_length/var": 65229.69921875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 2305468.0,
|
|
"eval_len_reward": 0.020261196419596672,
|
|
"eval_len_reward/correct": 0.1251496970653534,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1495969593524933,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.024701278656721115,
|
|
"eval_len_reward/correct/p75": 0.3076336979866028,
|
|
"eval_len_reward/correct/var": 0.060734037309885025,
|
|
"eval_len_reward/incorrect": -0.16090983152389526,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.08825862407684326,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.29965516924858093,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.033284444361925125,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.1348545253276825,
|
|
"eval_len_reward/p75": 0.21349772810935974,
|
|
"eval_len_reward/var": 0.06959052383899689,
|
|
"eval_mean_logprobs": -0.0152587890625,
|
|
"eval_mean_logprobs/var": 0.0001125335693359375,
|
|
"eval_missing_response_penalty": -0.10000000149011612,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.27272725105285645,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.20016680657863617,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.09030100703239441,
|
|
"eval_num_completions/total": 3900,
|
|
"eval_pred_value_log_probs": -0.6456806659698486,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6953125,
|
|
"eval_pred_value_log_probs/min": -0.9921875,
|
|
"eval_pred_value_log_probs/p25": -0.6953125,
|
|
"eval_pred_value_log_probs/p75": -0.6953125,
|
|
"eval_pred_value_log_probs/var": 0.02859373576939106,
|
|
"eval_reward": 0.6333333253860474,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23299884796142578,
|
|
"eval_rewards/accuracy_reward": 0.6333333253860474,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23299884796142578,
|
|
"eval_rewards/answer_logprob_reward": 3.0196218490600586,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1787933111190796,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.020261196419596672,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.1348545253276825,
|
|
"eval_rewards/len_reward/p75": 0.21349772810935974,
|
|
"eval_rewards/len_reward/var": 0.06959052383899689,
|
|
"eval_rewards/missing_response_penalty": -0.10000000149011612,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.09030100703239441,
|
|
"eval_runtime": 165.6153,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8315897583961487,
|
|
"eval_values/max": 1.9913291931152344,
|
|
"eval_values/median": -0.78662109375,
|
|
"eval_values/min": -2.000145435333252,
|
|
"eval_values/p25": -0.78662109375,
|
|
"eval_values/p75": -0.78662109375,
|
|
"eval_values/var": 0.1379442811012268,
|
|
"step": 13
|
|
},
|
|
{
|
|
"accuracy_reward": 0.78125,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.17179319262504578,
|
|
"adam_stats/lr_effective_max": 5.388263889472e-05,
|
|
"adam_stats/lr_effective_mean": -2.1835391827984552e-10,
|
|
"adam_stats/lr_effective_min": -5.37708583578933e-05,
|
|
"adam_stats/m_t_max": 0.003445042297244072,
|
|
"adam_stats/m_t_mean": -2.166033671868739e-11,
|
|
"adam_stats/m_t_min": -0.0031208600848913193,
|
|
"adam_stats/v_t_max": 2.2922347852727398e-05,
|
|
"adam_stats/v_t_mean": 1.4587817715946971e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.78125,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.17179319262504578,
|
|
"all_logprobs": -0.01267217192798853,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.125,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.017811747267842293,
|
|
"answer_logprob_reward": 3.0137572288513184,
|
|
"answer_logprob_reward/correct": 3.1133601665496826,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.04092986136674881,
|
|
"answer_logprob_reward/incorrect": 2.658031940460205,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.3449063003063202,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.14158384501934052,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 541.9583740234375,
|
|
"completion_length/correct": 475.7400207519531,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 441.0,
|
|
"completion_length/correct/min": 161.0,
|
|
"completion_length/correct/p25": 324.5,
|
|
"completion_length/correct/p75": 563.75,
|
|
"completion_length/correct/var": 42762.76171875,
|
|
"completion_length/incorrect": 778.452392578125,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 895.0,
|
|
"completion_length/incorrect/min": 261.0,
|
|
"completion_length/incorrect/p25": 518.25,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 73976.6953125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 473.0,
|
|
"completion_length/min": 161.0,
|
|
"completion_length/p25": 339.75,
|
|
"completion_length/p75": 705.25,
|
|
"completion_length/var": 64981.4765625,
|
|
"epoch": 0.35443037974683544,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1565290.0,
|
|
"grad_norm": 0.22427216172218323,
|
|
"kl": 0.01043701171875,
|
|
"learning_rate": 8.117449009293668e-06,
|
|
"len_reward": 0.08795583248138428,
|
|
"len_reward/correct": 0.17518459260463715,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.21158888936042786,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.03775645047426224,
|
|
"len_reward/correct/p75": 0.3615916669368744,
|
|
"len_reward/correct/var": 0.054440684616565704,
|
|
"len_reward/incorrect": -0.22357545793056488,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.20162558555603027,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.47105860710144043,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.046373918652534485,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.10566334426403046,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.0661887377500534,
|
|
"len_reward/p75": 0.3085714280605316,
|
|
"len_reward/var": 0.07974081486463547,
|
|
"loss": -0.7808,
|
|
"mean_logprobs": -0.01275634765625,
|
|
"mean_logprobs/var": 7.867813110351562e-05,
|
|
"missing_response_penalty": -0.0989583358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4523809552192688,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2537747025489807,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.08963242173194885,
|
|
"non_smoothed_logprobs/min": -18.0,
|
|
"num_completions/total": 2688,
|
|
"policy_loss": -0.7808328866958618,
|
|
"policy_loss/max": 0.0009762375266291201,
|
|
"policy_loss/median": -0.9995401501655579,
|
|
"policy_loss/min": -0.9998199343681335,
|
|
"policy_loss/p25": -0.9996650815010071,
|
|
"policy_loss/p75": -0.9993335604667664,
|
|
"policy_loss/var": 0.17179079353809357,
|
|
"pred_value_log_probs": -0.6513863801956177,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6953125,
|
|
"pred_value_log_probs/min": -1.1015625,
|
|
"pred_value_log_probs/p25": -0.6953125,
|
|
"pred_value_log_probs/p75": -0.6953125,
|
|
"pred_value_log_probs/var": 0.03190627694129944,
|
|
"reward": 0.78125,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.17179319262504578,
|
|
"rewards/accuracy_reward": 0.78125,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.17179319262504578,
|
|
"rewards/answer_logprob_reward": 3.0137572288513184,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.14158384501934052,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.08795583248138428,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.10566334426403046,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.0661887377500534,
|
|
"rewards/len_reward/p75": 0.3085714280605316,
|
|
"rewards/len_reward/var": 0.07974081486463547,
|
|
"rewards/missing_response_penalty": -0.0989583358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.08963242173194885,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 14,
|
|
"value_loss": 2.875,
|
|
"values": -0.7671071887016296,
|
|
"values/max": 1.9346656799316406,
|
|
"values/median": -0.78662109375,
|
|
"values/min": -1.9921550750732422,
|
|
"values/p25": -0.78662109375,
|
|
"values/p75": -0.78662109375,
|
|
"values/var": 0.2054801881313324
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 5.388263889472e-05,
|
|
"adam_stats/lr_effective_mean": -2.1835391827984552e-10,
|
|
"adam_stats/lr_effective_min": -5.37708583578933e-05,
|
|
"adam_stats/m_t_max": 0.003445042297244072,
|
|
"adam_stats/m_t_mean": -2.166033671868739e-11,
|
|
"adam_stats/m_t_min": -0.0031208600848913193,
|
|
"adam_stats/v_t_max": 2.2922347852727398e-05,
|
|
"adam_stats/v_t_mean": 1.4587817715946971e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.35443037974683544,
|
|
"eval_accuracy_reward": 0.6800000071525574,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.21832777559757233,
|
|
"eval_advantages": 0.6800000071525574,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.21832777559757233,
|
|
"eval_all_logprobs": -0.016139881685376167,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.90625,
|
|
"eval_all_logprobs/p25": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02557363547384739,
|
|
"eval_answer_logprob_reward": 3.0449326038360596,
|
|
"eval_answer_logprob_reward/correct": 3.1273913383483887,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06628566235303879,
|
|
"eval_answer_logprob_reward/incorrect": 2.8697075843811035,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.3330424129962921,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.16531668603420258,
|
|
"eval_completion_length": 548.9466552734375,
|
|
"eval_completion_length/correct": 483.0833435058594,
|
|
"eval_completion_length/correct/max": 1002.0,
|
|
"eval_completion_length/correct/median": 452.0,
|
|
"eval_completion_length/correct/min": 103.0,
|
|
"eval_completion_length/correct/p25": 339.25,
|
|
"eval_completion_length/correct/p75": 605.0,
|
|
"eval_completion_length/correct/var": 40634.56640625,
|
|
"eval_completion_length/incorrect": 688.90625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 644.0,
|
|
"eval_completion_length/incorrect/min": 200.0,
|
|
"eval_completion_length/incorrect/p25": 424.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 80034.359375,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 511.0,
|
|
"eval_completion_length/min": 103.0,
|
|
"eval_completion_length/p25": 352.0,
|
|
"eval_completion_length/p75": 686.5,
|
|
"eval_completion_length/var": 62266.02734375,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 2470152.0,
|
|
"eval_len_reward": 0.044923048466444016,
|
|
"eval_len_reward/correct": 0.14118969440460205,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.15497076511383057,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.005123578943312168,
|
|
"eval_len_reward/correct/p75": 0.30648672580718994,
|
|
"eval_len_reward/correct/var": 0.05321267619729042,
|
|
"eval_len_reward/incorrect": -0.15964357554912567,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.10513035953044891,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2941738963127136,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03156324848532677,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.006738544441759586,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.11079559475183487,
|
|
"eval_len_reward/p75": 0.22145964205265045,
|
|
"eval_len_reward/var": 0.06591493636369705,
|
|
"eval_mean_logprobs": -0.01544189453125,
|
|
"eval_mean_logprobs/var": 0.0003566741943359375,
|
|
"eval_missing_response_penalty": -0.09333333373069763,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2916666865348816,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.20877192914485931,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.084905244410038,
|
|
"eval_num_completions/total": 4200,
|
|
"eval_pred_value_log_probs": -0.6421944499015808,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.69140625,
|
|
"eval_pred_value_log_probs/min": -0.9375,
|
|
"eval_pred_value_log_probs/p25": -0.69140625,
|
|
"eval_pred_value_log_probs/p75": -0.69140625,
|
|
"eval_pred_value_log_probs/var": 0.0282747782766819,
|
|
"eval_reward": 0.6800000071525574,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.21832777559757233,
|
|
"eval_rewards/accuracy_reward": 0.6800000071525574,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.21832777559757233,
|
|
"eval_rewards/answer_logprob_reward": 3.0449326038360596,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.16531668603420258,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.044923048466444016,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.006738544441759586,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.11079559475183487,
|
|
"eval_rewards/len_reward/p75": 0.22145964205265045,
|
|
"eval_rewards/len_reward/var": 0.06591493636369705,
|
|
"eval_rewards/missing_response_penalty": -0.09333333373069763,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.084905244410038,
|
|
"eval_runtime": 176.2593,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8357735276222229,
|
|
"eval_values/max": 1.9913787841796875,
|
|
"eval_values/median": -0.7900390625,
|
|
"eval_values/min": -1.9919319152832031,
|
|
"eval_values/p25": -0.7900390625,
|
|
"eval_values/p75": -0.7900390625,
|
|
"eval_values/var": 0.1369415521621704,
|
|
"step": 14
|
|
},
|
|
{
|
|
"accuracy_reward": 0.78125,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.17179320752620697,
|
|
"adam_stats/lr_effective_max": 5.17590124218259e-05,
|
|
"adam_stats/lr_effective_mean": -1.4752990495914275e-10,
|
|
"adam_stats/lr_effective_min": -5.170961594558321e-05,
|
|
"adam_stats/m_t_max": 0.004034376237541437,
|
|
"adam_stats/m_t_mean": -2.961150075142527e-11,
|
|
"adam_stats/m_t_min": -0.002845394890755415,
|
|
"adam_stats/v_t_max": 2.2986629119259305e-05,
|
|
"adam_stats/v_t_mean": 1.4717971849945166e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.78125,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.17179320752620697,
|
|
"all_logprobs": -0.012538185343146324,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -10.625,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.019076447933912277,
|
|
"answer_logprob_reward": 2.933018207550049,
|
|
"answer_logprob_reward/correct": 3.085634469985962,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.055891625583171844,
|
|
"answer_logprob_reward/incorrect": 2.387960195541382,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/min": 1.5686159133911133,
|
|
"answer_logprob_reward/incorrect/p25": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.38665327429771423,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.5686159133911133,
|
|
"answer_logprob_reward/p25": 3.0047669410705566,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.21022039651870728,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 554.1979370117188,
|
|
"completion_length/correct": 472.3266906738281,
|
|
"completion_length/correct/max": 993.0,
|
|
"completion_length/correct/median": 429.0,
|
|
"completion_length/correct/min": 121.0,
|
|
"completion_length/correct/p25": 320.0,
|
|
"completion_length/correct/p75": 575.75,
|
|
"completion_length/correct/var": 38527.4296875,
|
|
"completion_length/incorrect": 846.5952758789062,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 313.0,
|
|
"completion_length/incorrect/p25": 709.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 60216.390625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 496.0,
|
|
"completion_length/min": 121.0,
|
|
"completion_length/p25": 345.75,
|
|
"completion_length/p75": 758.25,
|
|
"completion_length/var": 67045.7265625,
|
|
"epoch": 0.379746835443038,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1671696.0,
|
|
"grad_norm": 0.23879557847976685,
|
|
"kl": 0.010284423828125,
|
|
"learning_rate": 7.754484907260513e-06,
|
|
"len_reward": 0.07622233033180237,
|
|
"len_reward/correct": 0.16397397220134735,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.1973039209842682,
|
|
"len_reward/correct/min": -0.45950794219970703,
|
|
"len_reward/correct/p25": 0.03084648586809635,
|
|
"len_reward/correct/p75": 0.35621941089630127,
|
|
"len_reward/correct/var": 0.05613316223025322,
|
|
"len_reward/incorrect": -0.23717637360095978,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.2298821061849594,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.4051016569137573,
|
|
"len_reward/incorrect/p75": -0.0940435603260994,
|
|
"len_reward/incorrect/var": 0.03346768394112587,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.12388059496879578,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.11832349002361298,
|
|
"len_reward/p75": 0.30537301301956177,
|
|
"len_reward/var": 0.07861914485692978,
|
|
"loss": -0.7808,
|
|
"mean_logprobs": -0.012451171875,
|
|
"mean_logprobs/var": 8.869171142578125e-05,
|
|
"missing_response_penalty": -0.125,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.5714285969734192,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.25087106227874756,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.10994765162467957,
|
|
"non_smoothed_logprobs/min": -17.6875,
|
|
"num_completions/total": 2880,
|
|
"policy_loss": -0.7808372378349304,
|
|
"policy_loss/max": 0.0005718395113945007,
|
|
"policy_loss/median": -0.9995436668395996,
|
|
"policy_loss/min": -0.9998787045478821,
|
|
"policy_loss/p25": -0.9996510744094849,
|
|
"policy_loss/p75": -0.9992718696594238,
|
|
"policy_loss/var": 0.17176659405231476,
|
|
"pred_value_log_probs": -0.6482011079788208,
|
|
"pred_value_log_probs/max": -0.00087738037109375,
|
|
"pred_value_log_probs/median": -0.69140625,
|
|
"pred_value_log_probs/min": -1.3203125,
|
|
"pred_value_log_probs/p25": -0.69140625,
|
|
"pred_value_log_probs/p75": -0.69140625,
|
|
"pred_value_log_probs/var": 0.0415121465921402,
|
|
"reward": 0.78125,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.17179320752620697,
|
|
"rewards/accuracy_reward": 0.78125,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.17179320752620697,
|
|
"rewards/answer_logprob_reward": 2.933018207550049,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.5686159133911133,
|
|
"rewards/answer_logprob_reward/p25": 3.0047669410705566,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.21022039651870728,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.07622233033180237,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.12388059496879578,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.11832349002361298,
|
|
"rewards/len_reward/p75": 0.30537301301956177,
|
|
"rewards/len_reward/var": 0.07861914485692978,
|
|
"rewards/missing_response_penalty": -0.125,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.10994765162467957,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 15,
|
|
"value_loss": 2.6875,
|
|
"values": -0.7949997782707214,
|
|
"values/max": 1.22149658203125,
|
|
"values/median": -0.7900390625,
|
|
"values/min": -1.9999971389770508,
|
|
"values/p25": -0.7900390625,
|
|
"values/p75": -0.7900390625,
|
|
"values/var": 0.19552254676818848
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 5.17590124218259e-05,
|
|
"adam_stats/lr_effective_mean": -1.4752990495914275e-10,
|
|
"adam_stats/lr_effective_min": -5.170961594558321e-05,
|
|
"adam_stats/m_t_max": 0.004034376237541437,
|
|
"adam_stats/m_t_mean": -2.961150075142527e-11,
|
|
"adam_stats/m_t_min": -0.002845394890755415,
|
|
"adam_stats/v_t_max": 2.2986629119259305e-05,
|
|
"adam_stats/v_t_mean": 1.4717971849945166e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.379746835443038,
|
|
"eval_accuracy_reward": 0.6533333659172058,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22724638879299164,
|
|
"eval_advantages": 0.6533333659172058,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22724638879299164,
|
|
"eval_all_logprobs": -0.014098653569817543,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.375,
|
|
"eval_all_logprobs/p25": -1.1920928955078125e-07,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02137773297727108,
|
|
"eval_answer_logprob_reward": 3.027865409851074,
|
|
"eval_answer_logprob_reward/correct": 3.1033051013946533,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.08536802232265472,
|
|
"eval_answer_logprob_reward/incorrect": 2.8856899738311768,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.272965669631958,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.34138745069503784,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.272965669631958,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.18403808772563934,
|
|
"eval_completion_length": 546.8500366210938,
|
|
"eval_completion_length/correct": 476.3520202636719,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 442.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 305.0,
|
|
"eval_completion_length/correct/p75": 609.0,
|
|
"eval_completion_length/correct/var": 44579.4296875,
|
|
"eval_completion_length/incorrect": 679.7115478515625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 664.0,
|
|
"eval_completion_length/incorrect/min": 161.0,
|
|
"eval_completion_length/incorrect/p25": 398.75,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 86873.515625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 484.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 337.75,
|
|
"eval_completion_length/p75": 736.25,
|
|
"eval_completion_length/var": 68397.6484375,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 2634207.0,
|
|
"eval_len_reward": 0.05325581505894661,
|
|
"eval_len_reward/correct": 0.16086210310459137,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.19661563634872437,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.03216128051280975,
|
|
"eval_len_reward/correct/p75": 0.3326359987258911,
|
|
"eval_len_reward/correct/var": 0.047571588307619095,
|
|
"eval_len_reward/incorrect": -0.14954069256782532,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.04630269482731819,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2933231592178345,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03319944441318512,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.08192966878414154,
|
|
"eval_len_reward/p75": 0.2583830654621124,
|
|
"eval_len_reward/var": 0.0643567144870758,
|
|
"eval_mean_logprobs": -0.01409912109375,
|
|
"eval_mean_logprobs/var": 0.000118255615234375,
|
|
"eval_missing_response_penalty": -0.09000000357627869,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.25961539149284363,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.1940814107656479,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0821739137172699,
|
|
"eval_num_completions/total": 4500,
|
|
"eval_pred_value_log_probs": -0.6381974220275879,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6875,
|
|
"eval_pred_value_log_probs/min": -0.94921875,
|
|
"eval_pred_value_log_probs/p25": -0.6875,
|
|
"eval_pred_value_log_probs/p75": -0.6875,
|
|
"eval_pred_value_log_probs/var": 0.027867738157510757,
|
|
"eval_reward": 0.6533333659172058,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22724638879299164,
|
|
"eval_rewards/accuracy_reward": 0.6533333659172058,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22724638879299164,
|
|
"eval_rewards/answer_logprob_reward": 3.027865409851074,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.272965669631958,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.18403808772563934,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.05325581505894661,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.08192966878414154,
|
|
"eval_rewards/len_reward/p75": 0.2583830654621124,
|
|
"eval_rewards/len_reward/var": 0.0643567144870758,
|
|
"eval_rewards/missing_response_penalty": -0.09000000357627869,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0821739137172699,
|
|
"eval_runtime": 169.6683,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8470864295959473,
|
|
"eval_values/max": 1.9908599853515625,
|
|
"eval_values/median": -0.8037109375,
|
|
"eval_values/min": -1.9913673400878906,
|
|
"eval_values/p25": -0.8037109375,
|
|
"eval_values/p75": -0.8037109375,
|
|
"eval_values/var": 0.13635355234146118,
|
|
"step": 15
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7760416865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.1747109591960907,
|
|
"adam_stats/lr_effective_max": 4.9730995669960976e-05,
|
|
"adam_stats/lr_effective_mean": 2.7487906048562216e-11,
|
|
"adam_stats/lr_effective_min": -4.8396959755336866e-05,
|
|
"adam_stats/m_t_max": 0.004705157596617937,
|
|
"adam_stats/m_t_mean": -2.7418390474709398e-11,
|
|
"adam_stats/m_t_min": -0.0033178571611642838,
|
|
"adam_stats/v_t_max": 2.307903559994884e-05,
|
|
"adam_stats/v_t_mean": 1.4827556500326966e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7760416865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.1747109591960907,
|
|
"all_logprobs": -0.012429504655301571,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.5,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.018746089190244675,
|
|
"answer_logprob_reward": 2.9532697200775146,
|
|
"answer_logprob_reward/correct": 3.0818519592285156,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.08189358562231064,
|
|
"answer_logprob_reward/incorrect": 2.507716655731201,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.39231982827186584,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.2073162943124771,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 547.2864990234375,
|
|
"completion_length/correct": 464.0603942871094,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 408.0,
|
|
"completion_length/correct/min": 131.0,
|
|
"completion_length/correct/p25": 322.0,
|
|
"completion_length/correct/p75": 553.0,
|
|
"completion_length/correct/var": 41388.8671875,
|
|
"completion_length/incorrect": 835.6744384765625,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 267.0,
|
|
"completion_length/incorrect/p25": 634.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 65544.5546875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 454.0,
|
|
"completion_length/min": 131.0,
|
|
"completion_length/p25": 346.25,
|
|
"completion_length/p75": 718.75,
|
|
"completion_length/var": 70610.9453125,
|
|
"epoch": 0.4050632911392405,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1776775.0,
|
|
"grad_norm": 0.23507139086723328,
|
|
"kl": 0.0107421875,
|
|
"learning_rate": 7.369343312364994e-06,
|
|
"len_reward": 0.09303848445415497,
|
|
"len_reward/correct": 0.19003821909427643,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.23529411852359772,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.09877175092697144,
|
|
"len_reward/correct/p75": 0.3461134433746338,
|
|
"len_reward/correct/var": 0.04861581698060036,
|
|
"len_reward/incorrect": -0.2430768609046936,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.26805612444877625,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3951869010925293,
|
|
"len_reward/incorrect/p75": -0.06606549769639969,
|
|
"len_reward/incorrect/var": 0.032611481845378876,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.1441424936056137,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.0984899252653122,
|
|
"len_reward/p75": 0.3025240898132324,
|
|
"len_reward/var": 0.07761579006910324,
|
|
"loss": -0.7756,
|
|
"mean_logprobs": -0.01239013671875,
|
|
"mean_logprobs/var": 8.7738037109375e-05,
|
|
"missing_response_penalty": -0.1197916716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.5348837375640869,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2547065317630768,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.10599368065595627,
|
|
"non_smoothed_logprobs/min": -17.3125,
|
|
"num_completions/total": 3072,
|
|
"policy_loss": -0.7756123542785645,
|
|
"policy_loss/max": 0.0010877820895984769,
|
|
"policy_loss/median": -0.9995073676109314,
|
|
"policy_loss/min": -0.9998544454574585,
|
|
"policy_loss/p25": -0.9996453523635864,
|
|
"policy_loss/p75": -0.999257504940033,
|
|
"policy_loss/var": 0.17469975352287292,
|
|
"pred_value_log_probs": -0.6218863129615784,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6875,
|
|
"pred_value_log_probs/min": -1.2734375,
|
|
"pred_value_log_probs/p25": -0.6875,
|
|
"pred_value_log_probs/p75": -0.6875,
|
|
"pred_value_log_probs/var": 0.055848974734544754,
|
|
"reward": 0.7760416865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.1747109591960907,
|
|
"rewards/accuracy_reward": 0.7760416865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.1747109591960907,
|
|
"rewards/answer_logprob_reward": 2.9532697200775146,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.2073162943124771,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09303848445415497,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.1441424936056137,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.0984899252653122,
|
|
"rewards/len_reward/p75": 0.3025240898132324,
|
|
"rewards/len_reward/var": 0.07761579006910324,
|
|
"rewards/missing_response_penalty": -0.1197916716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.10599368065595627,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 16,
|
|
"value_loss": 3.2265625,
|
|
"values": -0.7615966796875,
|
|
"values/max": 1.9897232055664062,
|
|
"values/median": -0.8037109375,
|
|
"values/min": -2.001072883605957,
|
|
"values/p25": -0.8037109375,
|
|
"values/p75": -0.8037109375,
|
|
"values/var": 0.32145166397094727
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 4.9730995669960976e-05,
|
|
"adam_stats/lr_effective_mean": 2.7487906048562216e-11,
|
|
"adam_stats/lr_effective_min": -4.8396959755336866e-05,
|
|
"adam_stats/m_t_max": 0.004705157596617937,
|
|
"adam_stats/m_t_mean": -2.7418390474709398e-11,
|
|
"adam_stats/m_t_min": -0.0033178571611642838,
|
|
"adam_stats/v_t_max": 2.307903559994884e-05,
|
|
"adam_stats/v_t_mean": 1.4827556500326966e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.4050632911392405,
|
|
"eval_accuracy_reward": 0.6600000262260437,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22515049576759338,
|
|
"eval_advantages": 0.6600000262260437,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22515049576759338,
|
|
"eval_all_logprobs": -0.013608547858893871,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -10.0,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.021750587970018387,
|
|
"eval_answer_logprob_reward": 3.0337154865264893,
|
|
"eval_answer_logprob_reward/correct": 3.1085681915283203,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07818499207496643,
|
|
"eval_answer_logprob_reward/incorrect": 2.888413429260254,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.24697057902812958,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.14585068821907043,
|
|
"eval_completion_length": 551.8099975585938,
|
|
"eval_completion_length/correct": 485.5404052734375,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 444.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 322.25,
|
|
"eval_completion_length/correct/p75": 637.0,
|
|
"eval_completion_length/correct/var": 45999.62890625,
|
|
"eval_completion_length/incorrect": 680.4509887695312,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 638.0,
|
|
"eval_completion_length/incorrect/min": 172.0,
|
|
"eval_completion_length/incorrect/p25": 433.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 82459.515625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 489.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 344.75,
|
|
"eval_completion_length/p75": 733.25,
|
|
"eval_completion_length/var": 66715.1640625,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 2799750.0,
|
|
"eval_len_reward": 0.04988803341984749,
|
|
"eval_len_reward/correct": 0.15069575607776642,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1733633577823639,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.002489530947059393,
|
|
"eval_len_reward/correct/p75": 0.3220340609550476,
|
|
"eval_len_reward/correct/var": 0.05049169436097145,
|
|
"eval_len_reward/incorrect": -0.14579756557941437,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.041744664311409,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.3054758310317993,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03288053721189499,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.09548091888427734,
|
|
"eval_len_reward/p75": 0.23046810925006866,
|
|
"eval_len_reward/var": 0.06416650861501694,
|
|
"eval_mean_logprobs": -0.01409912109375,
|
|
"eval_mean_logprobs/var": 9.72747802734375e-05,
|
|
"eval_missing_response_penalty": -0.09666667133569717,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.28431373834609985,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.20549406111240387,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0876142606139183,
|
|
"eval_num_completions/total": 4800,
|
|
"eval_pred_value_log_probs": -0.6389652490615845,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6875,
|
|
"eval_pred_value_log_probs/min": -0.91015625,
|
|
"eval_pred_value_log_probs/p25": -0.6875,
|
|
"eval_pred_value_log_probs/p75": -0.6875,
|
|
"eval_pred_value_log_probs/var": 0.027192160487174988,
|
|
"eval_reward": 0.6600000262260437,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22515049576759338,
|
|
"eval_rewards/accuracy_reward": 0.6600000262260437,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22515049576759338,
|
|
"eval_rewards/answer_logprob_reward": 3.0337154865264893,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.14585068821907043,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.04988803341984749,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.09548091888427734,
|
|
"eval_rewards/len_reward/p75": 0.23046810925006866,
|
|
"eval_rewards/len_reward/var": 0.06416650861501694,
|
|
"eval_rewards/missing_response_penalty": -0.09666667133569717,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0876142606139183,
|
|
"eval_runtime": 165.2241,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8467819094657898,
|
|
"eval_values/max": 1.99127197265625,
|
|
"eval_values/median": -0.8046875,
|
|
"eval_values/min": -1.990447998046875,
|
|
"eval_values/p25": -0.8046875,
|
|
"eval_values/p75": -0.8046875,
|
|
"eval_values/var": 0.1333002895116806,
|
|
"step": 16
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7135416865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.20547011494636536,
|
|
"adam_stats/lr_effective_max": 4.561969035421498e-05,
|
|
"adam_stats/lr_effective_mean": 3.8943893249099304e-11,
|
|
"adam_stats/lr_effective_min": -4.658138641389087e-05,
|
|
"adam_stats/m_t_max": 0.004057639744132757,
|
|
"adam_stats/m_t_mean": -1.9962500402703753e-11,
|
|
"adam_stats/m_t_min": -0.0026576807722449303,
|
|
"adam_stats/v_t_max": 2.3059088562149554e-05,
|
|
"adam_stats/v_t_mean": 1.4879517973645506e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7135416865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.20547011494636536,
|
|
"all_logprobs": -0.014197630807757378,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -8.75,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.022814959287643433,
|
|
"answer_logprob_reward": 2.971250057220459,
|
|
"answer_logprob_reward/correct": 3.0861237049102783,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06222710758447647,
|
|
"answer_logprob_reward/incorrect": 2.6851089000701904,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.3862943649291992,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.426470547914505,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.3862943649291992,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.19792331755161285,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 513.0208740234375,
|
|
"completion_length/correct": 430.98541259765625,
|
|
"completion_length/correct/max": 943.0,
|
|
"completion_length/correct/median": 411.0,
|
|
"completion_length/correct/min": 153.0,
|
|
"completion_length/correct/p25": 275.0,
|
|
"completion_length/correct/p75": 552.0,
|
|
"completion_length/correct/var": 37020.87890625,
|
|
"completion_length/incorrect": 717.3635864257812,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 715.0,
|
|
"completion_length/incorrect/min": 237.0,
|
|
"completion_length/incorrect/p25": 474.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 70372.3125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 460.0,
|
|
"completion_length/min": 153.0,
|
|
"completion_length/p25": 298.5,
|
|
"completion_length/p75": 665.25,
|
|
"completion_length/var": 63107.37109375,
|
|
"epoch": 0.43037974683544306,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1875275.0,
|
|
"grad_norm": 0.2200733721256256,
|
|
"kl": 0.012603759765625,
|
|
"learning_rate": 6.965125158269619e-06,
|
|
"len_reward": 0.07363046705722809,
|
|
"len_reward/correct": 0.16898098587989807,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2068466693162918,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.022179363295435905,
|
|
"len_reward/correct/p75": 0.350384384393692,
|
|
"len_reward/correct/var": 0.05886177346110344,
|
|
"len_reward/incorrect": -0.16387902200222015,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.07846952229738235,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.2992038130760193,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03208499401807785,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.061749570071697235,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.09734134376049042,
|
|
"len_reward/p75": 0.3237048089504242,
|
|
"len_reward/var": 0.07374843209981918,
|
|
"loss": -0.713,
|
|
"mean_logprobs": -0.01446533203125,
|
|
"mean_logprobs/var": 9.965896606445312e-05,
|
|
"missing_response_penalty": -0.0989583358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.34545454382896423,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.23030301928520203,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.08963242173194885,
|
|
"non_smoothed_logprobs/min": -17.6875,
|
|
"num_completions/total": 3264,
|
|
"policy_loss": -0.7130376100540161,
|
|
"policy_loss/max": 0.001001933473162353,
|
|
"policy_loss/median": -0.9994107484817505,
|
|
"policy_loss/min": -0.9998096227645874,
|
|
"policy_loss/p25": -0.9995970726013184,
|
|
"policy_loss/p75": 0.00025670218747109175,
|
|
"policy_loss/var": 0.20545172691345215,
|
|
"pred_value_log_probs": -0.6223148107528687,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6875,
|
|
"pred_value_log_probs/min": -1.1953125,
|
|
"pred_value_log_probs/p25": -0.6875,
|
|
"pred_value_log_probs/p75": -0.6875,
|
|
"pred_value_log_probs/var": 0.03758833557367325,
|
|
"reward": 0.7135416865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.20547011494636536,
|
|
"rewards/accuracy_reward": 0.7135416865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.20547011494636536,
|
|
"rewards/answer_logprob_reward": 2.971250057220459,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.3862943649291992,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.19792331755161285,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.07363046705722809,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.061749570071697235,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.09734134376049042,
|
|
"rewards/len_reward/p75": 0.3237048089504242,
|
|
"rewards/len_reward/var": 0.07374843209981918,
|
|
"rewards/missing_response_penalty": -0.0989583358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.08963242173194885,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 17,
|
|
"value_loss": 2.796875,
|
|
"values": -0.8071715235710144,
|
|
"values/max": 1.830169677734375,
|
|
"values/median": -0.8046875,
|
|
"values/min": -1.9844542741775513,
|
|
"values/p25": -0.8046875,
|
|
"values/p75": -0.8046875,
|
|
"values/var": 0.2470327913761139
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 4.561969035421498e-05,
|
|
"adam_stats/lr_effective_mean": 3.8943893249099304e-11,
|
|
"adam_stats/lr_effective_min": -4.658138641389087e-05,
|
|
"adam_stats/m_t_max": 0.004057639744132757,
|
|
"adam_stats/m_t_mean": -1.9962500402703753e-11,
|
|
"adam_stats/m_t_min": -0.0026576807722449303,
|
|
"adam_stats/v_t_max": 2.3059088562149554e-05,
|
|
"adam_stats/v_t_mean": 1.4879517973645506e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.43037974683544306,
|
|
"eval_accuracy_reward": 0.643333375453949,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23022297024726868,
|
|
"eval_advantages": 0.643333375453949,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23022297024726868,
|
|
"eval_all_logprobs": -0.014583272859454155,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -11.0,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02367628738284111,
|
|
"eval_answer_logprob_reward": 3.040311574935913,
|
|
"eval_answer_logprob_reward/correct": 3.108691930770874,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.08312822133302689,
|
|
"eval_answer_logprob_reward/incorrect": 2.91697096824646,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.29633092880249023,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1668960005044937,
|
|
"eval_completion_length": 538.5833740234375,
|
|
"eval_completion_length/correct": 470.3575134277344,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 434.0,
|
|
"eval_completion_length/correct/min": 123.0,
|
|
"eval_completion_length/correct/p25": 315.0,
|
|
"eval_completion_length/correct/p75": 574.0,
|
|
"eval_completion_length/correct/var": 41447.609375,
|
|
"eval_completion_length/incorrect": 661.6448364257812,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 637.0,
|
|
"eval_completion_length/incorrect/min": 172.0,
|
|
"eval_completion_length/incorrect/p25": 423.5,
|
|
"eval_completion_length/incorrect/p75": 995.0,
|
|
"eval_completion_length/incorrect/var": 80487.4375,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 474.0,
|
|
"eval_completion_length/min": 123.0,
|
|
"eval_completion_length/p25": 330.75,
|
|
"eval_completion_length/p75": 704.0,
|
|
"eval_completion_length/var": 63573.23828125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 2961325.0,
|
|
"eval_len_reward": 0.022688377648591995,
|
|
"eval_len_reward/correct": 0.12160007655620575,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1525157243013382,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.034984275698661804,
|
|
"eval_len_reward/correct/p75": 0.28127622604370117,
|
|
"eval_len_reward/correct/var": 0.058058690279722214,
|
|
"eval_len_reward/incorrect": -0.1557224690914154,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.06534695625305176,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2915356159210205,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03322194516658783,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.15201473236083984,
|
|
"eval_len_reward/p75": 0.210999995470047,
|
|
"eval_len_reward/var": 0.0667654499411583,
|
|
"eval_mean_logprobs": -0.0146484375,
|
|
"eval_mean_logprobs/var": 0.00010395050048828125,
|
|
"eval_missing_response_penalty": -0.0833333358168602,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.23364485800266266,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.18074415624141693,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_num_completions/total": 5100,
|
|
"eval_pred_value_log_probs": -0.6259163618087769,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.671875,
|
|
"eval_pred_value_log_probs/min": -0.94921875,
|
|
"eval_pred_value_log_probs/p25": -0.671875,
|
|
"eval_pred_value_log_probs/p75": -0.671875,
|
|
"eval_pred_value_log_probs/var": 0.025486089289188385,
|
|
"eval_reward": 0.643333375453949,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23022297024726868,
|
|
"eval_rewards/accuracy_reward": 0.643333375453949,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23022297024726868,
|
|
"eval_rewards/answer_logprob_reward": 3.040311574935913,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1668960005044937,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.022688377648591995,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.15201473236083984,
|
|
"eval_rewards/len_reward/p75": 0.210999995470047,
|
|
"eval_rewards/len_reward/var": 0.0667654499411583,
|
|
"eval_rewards/missing_response_penalty": -0.0833333358168602,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_runtime": 163.4822,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8655804395675659,
|
|
"eval_values/max": 1.990936279296875,
|
|
"eval_values/median": -0.8271484375,
|
|
"eval_values/min": -1.9853706359863281,
|
|
"eval_values/p25": -0.8271484375,
|
|
"eval_values/p75": -0.8271484375,
|
|
"eval_values/var": 0.12937664985656738,
|
|
"step": 17
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7552083730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.18583661317825317,
|
|
"adam_stats/lr_effective_max": 4.375962816993706e-05,
|
|
"adam_stats/lr_effective_mean": 2.8192944506733042e-11,
|
|
"adam_stats/lr_effective_min": -4.2835705244215205e-05,
|
|
"adam_stats/m_t_max": 0.005055684596300125,
|
|
"adam_stats/m_t_mean": -6.154668441560318e-11,
|
|
"adam_stats/m_t_min": -0.0036076109390705824,
|
|
"adam_stats/v_t_max": 2.3233094907482155e-05,
|
|
"adam_stats/v_t_mean": 1.5197829973667698e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7552083730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.18583661317825317,
|
|
"all_logprobs": -0.013125843368470669,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.5,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.02088928036391735,
|
|
"answer_logprob_reward": 2.927567481994629,
|
|
"answer_logprob_reward/correct": 3.0681064128875732,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06456664204597473,
|
|
"answer_logprob_reward/incorrect": 2.493990182876587,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/min": 1.3862943649291992,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.5105509757995605,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.3862943649291992,
|
|
"answer_logprob_reward/p25": 3.0047669410705566,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.23289190232753754,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 553.984375,
|
|
"completion_length/correct": 463.2896423339844,
|
|
"completion_length/correct/max": 1017.0,
|
|
"completion_length/correct/median": 418.0,
|
|
"completion_length/correct/min": 143.0,
|
|
"completion_length/correct/p25": 334.0,
|
|
"completion_length/correct/p75": 561.0,
|
|
"completion_length/correct/var": 37730.87109375,
|
|
"completion_length/incorrect": 833.7872314453125,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 244.0,
|
|
"completion_length/incorrect/p25": 661.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 62925.0390625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 472.0,
|
|
"completion_length/min": 143.0,
|
|
"completion_length/p25": 353.0,
|
|
"completion_length/p75": 737.25,
|
|
"completion_length/var": 69110.5390625,
|
|
"epoch": 0.45569620253164556,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 1981640.0,
|
|
"grad_norm": 0.31346508860588074,
|
|
"kl": 0.011932373046875,
|
|
"learning_rate": 6.545084971874738e-06,
|
|
"len_reward": 0.06259474903345108,
|
|
"len_reward/correct": 0.16533038020133972,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2032332569360733,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.04145798459649086,
|
|
"len_reward/correct/p75": 0.3477851152420044,
|
|
"len_reward/correct/var": 0.061689212918281555,
|
|
"len_reward/incorrect": -0.25435560941696167,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.24641422927379608,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.4109390676021576,
|
|
"len_reward/incorrect/p75": -0.1067853569984436,
|
|
"len_reward/incorrect/var": 0.03014647774398327,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.08783120661973953,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.1487763375043869,
|
|
"len_reward/p75": 0.2968714237213135,
|
|
"len_reward/var": 0.08650212734937668,
|
|
"loss": -0.7547,
|
|
"mean_logprobs": -0.01318359375,
|
|
"mean_logprobs/var": 0.0001850128173828125,
|
|
"missing_response_penalty": -0.125,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.5106382966041565,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.25531914830207825,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.10994765162467957,
|
|
"non_smoothed_logprobs/min": -17.625,
|
|
"num_completions/total": 3456,
|
|
"policy_loss": -0.754731297492981,
|
|
"policy_loss/max": 0.0008864151895977557,
|
|
"policy_loss/median": -0.99945467710495,
|
|
"policy_loss/min": -0.9998028874397278,
|
|
"policy_loss/p25": -0.999605655670166,
|
|
"policy_loss/p75": -0.9989223480224609,
|
|
"policy_loss/var": 0.18581964075565338,
|
|
"pred_value_log_probs": -0.6210931539535522,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.671875,
|
|
"pred_value_log_probs/min": -1.421875,
|
|
"pred_value_log_probs/p25": -0.671875,
|
|
"pred_value_log_probs/p75": -0.671875,
|
|
"pred_value_log_probs/var": 0.04601706191897392,
|
|
"reward": 0.7552083730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.18583661317825317,
|
|
"rewards/accuracy_reward": 0.7552083730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.18583661317825317,
|
|
"rewards/answer_logprob_reward": 2.927567481994629,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.3862943649291992,
|
|
"rewards/answer_logprob_reward/p25": 3.0047669410705566,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.23289190232753754,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.06259474903345108,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.08783120661973953,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.1487763375043869,
|
|
"rewards/len_reward/p75": 0.2968714237213135,
|
|
"rewards/len_reward/var": 0.08650212734937668,
|
|
"rewards/missing_response_penalty": -0.125,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.10994765162467957,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 18,
|
|
"value_loss": 2.96875,
|
|
"values": -0.8621608018875122,
|
|
"values/max": 2.0,
|
|
"values/median": -0.8271484375,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.8271484375,
|
|
"values/p75": -0.8271484375,
|
|
"values/var": 0.20169949531555176
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 4.375962816993706e-05,
|
|
"adam_stats/lr_effective_mean": 2.8192944506733042e-11,
|
|
"adam_stats/lr_effective_min": -4.2835705244215205e-05,
|
|
"adam_stats/m_t_max": 0.005055684596300125,
|
|
"adam_stats/m_t_mean": -6.154668441560318e-11,
|
|
"adam_stats/m_t_min": -0.0036076109390705824,
|
|
"adam_stats/v_t_max": 2.3233094907482155e-05,
|
|
"adam_stats/v_t_mean": 1.5197829973667698e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.45569620253164556,
|
|
"eval_accuracy_reward": 0.6399999856948853,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23117054998874664,
|
|
"eval_advantages": 0.6399999856948853,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23117054998874664,
|
|
"eval_all_logprobs": -0.014533042907714844,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -12.875,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.023786023259162903,
|
|
"eval_answer_logprob_reward": 3.0196218490600586,
|
|
"eval_answer_logprob_reward/correct": 3.1066195964813232,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.08187548816204071,
|
|
"eval_answer_logprob_reward/incorrect": 2.864959478378296,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.35166385769844055,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1916482299566269,
|
|
"eval_completion_length": 538.77001953125,
|
|
"eval_completion_length/correct": 471.7864685058594,
|
|
"eval_completion_length/correct/max": 1021.0,
|
|
"eval_completion_length/correct/median": 434.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 312.5,
|
|
"eval_completion_length/correct/p75": 603.25,
|
|
"eval_completion_length/correct/var": 43407.1328125,
|
|
"eval_completion_length/incorrect": 657.8518676757812,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 593.0,
|
|
"eval_completion_length/incorrect/min": 163.0,
|
|
"eval_completion_length/incorrect/p25": 432.5,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 80225.25,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 491.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 339.75,
|
|
"eval_completion_length/p75": 686.75,
|
|
"eval_completion_length/var": 64440.875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 3122956.0,
|
|
"eval_len_reward": 0.04496534913778305,
|
|
"eval_len_reward/correct": 0.1389261931180954,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.16355518996715546,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.011785062029957771,
|
|
"eval_len_reward/correct/p75": 0.31801870465278625,
|
|
"eval_len_reward/correct/var": 0.05499494448304176,
|
|
"eval_len_reward/incorrect": -0.12207616120576859,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.0275835283100605,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.20396654307842255,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.027708251029253006,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.08302466571331024,
|
|
"eval_len_reward/p75": 0.23195244371891022,
|
|
"eval_len_reward/var": 0.06079406663775444,
|
|
"eval_mean_logprobs": -0.0146484375,
|
|
"eval_mean_logprobs/var": 0.0001277923583984375,
|
|
"eval_missing_response_penalty": -0.09666667133569717,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.26851850748062134,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.19825200736522675,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0876142606139183,
|
|
"eval_num_completions/total": 5400,
|
|
"eval_pred_value_log_probs": -0.6362558007240295,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.68359375,
|
|
"eval_pred_value_log_probs/min": -0.96875,
|
|
"eval_pred_value_log_probs/p25": -0.68359375,
|
|
"eval_pred_value_log_probs/p75": -0.68359375,
|
|
"eval_pred_value_log_probs/var": 0.026474738493561745,
|
|
"eval_reward": 0.6399999856948853,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23117054998874664,
|
|
"eval_rewards/accuracy_reward": 0.6399999856948853,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23117054998874664,
|
|
"eval_rewards/answer_logprob_reward": 3.0196218490600586,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1916482299566269,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.04496534913778305,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.08302466571331024,
|
|
"eval_rewards/len_reward/p75": 0.23195244371891022,
|
|
"eval_rewards/len_reward/var": 0.06079406663775444,
|
|
"eval_rewards/missing_response_penalty": -0.09666667133569717,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0876142606139183,
|
|
"eval_runtime": 168.3289,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8485709428787231,
|
|
"eval_values/max": 1.9911270141601562,
|
|
"eval_values/median": -0.80859375,
|
|
"eval_values/min": -1.9788665771484375,
|
|
"eval_values/p25": -0.80859375,
|
|
"eval_values/p75": -0.80859375,
|
|
"eval_values/var": 0.129899263381958,
|
|
"step": 18
|
|
},
|
|
{
|
|
"accuracy_reward": 0.8020833730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.15957678854465485,
|
|
"adam_stats/lr_effective_max": 3.9845628634793684e-05,
|
|
"adam_stats/lr_effective_mean": 2.0078984308558034e-10,
|
|
"adam_stats/lr_effective_min": -3.964529969380237e-05,
|
|
"adam_stats/m_t_max": 0.00423883693292737,
|
|
"adam_stats/m_t_mean": -4.4684693445429247e-11,
|
|
"adam_stats/m_t_min": -0.003130326746031642,
|
|
"adam_stats/v_t_max": 2.3219552531372756e-05,
|
|
"adam_stats/v_t_mean": 1.53001916691764e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.8020833730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.15957678854465485,
|
|
"all_logprobs": -0.01230152789503336,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.84375,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.018796944990754128,
|
|
"answer_logprob_reward": 3.048903226852417,
|
|
"answer_logprob_reward/correct": 3.123277425765991,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.04101899638772011,
|
|
"answer_logprob_reward/incorrect": 2.747490644454956,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.272623747587204,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 2.079441547393799,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.10820484906435013,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 529.3125,
|
|
"completion_length/correct": 455.3246765136719,
|
|
"completion_length/correct/max": 913.0,
|
|
"completion_length/correct/median": 423.0,
|
|
"completion_length/correct/min": 146.0,
|
|
"completion_length/correct/p25": 313.25,
|
|
"completion_length/correct/p75": 583.25,
|
|
"completion_length/correct/var": 33110.40234375,
|
|
"completion_length/incorrect": 829.1578979492188,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1003.0,
|
|
"completion_length/incorrect/min": 350.0,
|
|
"completion_length/incorrect/p25": 621.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 59654.7890625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 466.0,
|
|
"completion_length/min": 146.0,
|
|
"completion_length/p25": 338.0,
|
|
"completion_length/p75": 675.25,
|
|
"completion_length/var": 60380.21484375,
|
|
"epoch": 0.4810126582278481,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2083268.0,
|
|
"grad_norm": 0.24223755300045013,
|
|
"kl": 0.01190185546875,
|
|
"learning_rate": 6.112604669781572e-06,
|
|
"len_reward": 0.09958770126104355,
|
|
"len_reward/correct": 0.18344828486442566,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2180722951889038,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.020527295768260956,
|
|
"len_reward/correct/p75": 0.3609812259674072,
|
|
"len_reward/correct/var": 0.04795420542359352,
|
|
"len_reward/incorrect": -0.24026833474636078,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.2524407207965851,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.462963342666626,
|
|
"len_reward/incorrect/p75": -0.009423407725989819,
|
|
"len_reward/incorrect/var": 0.04041590169072151,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.12857143580913544,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.06484229117631912,
|
|
"len_reward/p75": 0.3338591456413269,
|
|
"len_reward/var": 0.07489258050918579,
|
|
"loss": -0.8016,
|
|
"mean_logprobs": -0.01177978515625,
|
|
"mean_logprobs/var": 8.440017700195312e-05,
|
|
"missing_response_penalty": -0.0833333358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.42105263471603394,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.25035563111305237,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.07678882777690887,
|
|
"non_smoothed_logprobs/min": -17.25,
|
|
"num_completions/total": 3648,
|
|
"policy_loss": -0.8016080260276794,
|
|
"policy_loss/max": 0.0008301552152261138,
|
|
"policy_loss/median": -0.9994857907295227,
|
|
"policy_loss/min": -0.9998277425765991,
|
|
"policy_loss/p25": -0.9996201395988464,
|
|
"policy_loss/p75": -0.9992146492004395,
|
|
"policy_loss/var": 0.15955623984336853,
|
|
"pred_value_log_probs": -0.634164035320282,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.68359375,
|
|
"pred_value_log_probs/min": -1.3125,
|
|
"pred_value_log_probs/p25": -0.68359375,
|
|
"pred_value_log_probs/p75": -0.68359375,
|
|
"pred_value_log_probs/var": 0.04011666402220726,
|
|
"reward": 0.8020833730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.15957678854465485,
|
|
"rewards/accuracy_reward": 0.8020833730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.15957678854465485,
|
|
"rewards/answer_logprob_reward": 3.048903226852417,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 2.079441547393799,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.10820484906435013,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09958770126104355,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.12857143580913544,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.06484229117631912,
|
|
"rewards/len_reward/p75": 0.3338591456413269,
|
|
"rewards/len_reward/var": 0.07489258050918579,
|
|
"rewards/missing_response_penalty": -0.0833333358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.07678882777690887,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 19,
|
|
"value_loss": 2.953125,
|
|
"values": -0.7967839241027832,
|
|
"values/max": 2.0,
|
|
"values/median": -0.80859375,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.80859375,
|
|
"values/p75": -0.80859375,
|
|
"values/var": 0.2700471878051758
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 3.9845628634793684e-05,
|
|
"adam_stats/lr_effective_mean": 2.0078984308558034e-10,
|
|
"adam_stats/lr_effective_min": -3.964529969380237e-05,
|
|
"adam_stats/m_t_max": 0.00423883693292737,
|
|
"adam_stats/m_t_mean": -4.4684693445429247e-11,
|
|
"adam_stats/m_t_min": -0.003130326746031642,
|
|
"adam_stats/v_t_max": 2.3219552531372756e-05,
|
|
"adam_stats/v_t_mean": 1.53001916691764e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.4810126582278481,
|
|
"eval_accuracy_reward": 0.6866666674613953,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.21587511897087097,
|
|
"eval_advantages": 0.6866666674613953,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.21587511897087097,
|
|
"eval_all_logprobs": -0.013197014108300209,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -12.5,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.019858872517943382,
|
|
"eval_answer_logprob_reward": 3.0339996814727783,
|
|
"eval_answer_logprob_reward/correct": 3.1187570095062256,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06618177890777588,
|
|
"eval_answer_logprob_reward/incorrect": 2.848254919052124,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.272965669631958,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.3835936486721039,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.272965669631958,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.18048304319381714,
|
|
"eval_completion_length": 546.316650390625,
|
|
"eval_completion_length/correct": 492.4563293457031,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 452.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 313.5,
|
|
"eval_completion_length/correct/p75": 630.25,
|
|
"eval_completion_length/correct/var": 49226.74609375,
|
|
"eval_completion_length/incorrect": 664.3510131835938,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 637.0,
|
|
"eval_completion_length/incorrect/min": 170.0,
|
|
"eval_completion_length/incorrect/p25": 403.75,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 88437.84375,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 496.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 332.75,
|
|
"eval_completion_length/p75": 720.0,
|
|
"eval_completion_length/var": 67636.8359375,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 3286851.0,
|
|
"eval_len_reward": 0.03408301994204521,
|
|
"eval_len_reward/correct": 0.1176217645406723,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.15866558253765106,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.02469135820865631,
|
|
"eval_len_reward/correct/p75": 0.30703064799308777,
|
|
"eval_len_reward/correct/var": 0.06151648238301277,
|
|
"eval_len_reward/incorrect": -0.14899122714996338,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.07014767825603485,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2524755299091339,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.031159698963165283,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.11927083134651184,
|
|
"eval_len_reward/p75": 0.221572607755661,
|
|
"eval_len_reward/var": 0.06721360236406326,
|
|
"eval_mean_logprobs": -0.01361083984375,
|
|
"eval_mean_logprobs/var": 0.00014781951904296875,
|
|
"eval_missing_response_penalty": -0.08666666597127914,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.27659574151039124,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.20224204659461975,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0794202908873558,
|
|
"eval_num_completions/total": 5700,
|
|
"eval_pred_value_log_probs": -0.6355226039886475,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.68359375,
|
|
"eval_pred_value_log_probs/min": -0.953125,
|
|
"eval_pred_value_log_probs/p25": -0.68359375,
|
|
"eval_pred_value_log_probs/p75": -0.68359375,
|
|
"eval_pred_value_log_probs/var": 0.026992160826921463,
|
|
"eval_reward": 0.6866666674613953,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.21587511897087097,
|
|
"eval_rewards/accuracy_reward": 0.6866666674613953,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.21587511897087097,
|
|
"eval_rewards/answer_logprob_reward": 3.0339996814727783,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.272965669631958,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.18048304319381714,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.03408301994204521,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.11927083134651184,
|
|
"eval_rewards/len_reward/p75": 0.221572607755661,
|
|
"eval_rewards/len_reward/var": 0.06721360236406326,
|
|
"eval_rewards/missing_response_penalty": -0.08666666597127914,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0794202908873558,
|
|
"eval_runtime": 176.7268,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8475617170333862,
|
|
"eval_values/max": 1.9909133911132812,
|
|
"eval_values/median": -0.8076171875,
|
|
"eval_values/min": -1.98785400390625,
|
|
"eval_values/p25": -0.8076171875,
|
|
"eval_values/p75": -0.8076171875,
|
|
"eval_values/var": 0.13097244501113892,
|
|
"step": 19
|
|
},
|
|
{
|
|
"accuracy_reward": 0.703125,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.20983311533927917,
|
|
"adam_stats/lr_effective_max": 3.718593143275939e-05,
|
|
"adam_stats/lr_effective_mean": 1.5384329921097617e-10,
|
|
"adam_stats/lr_effective_min": -3.6357450881041586e-05,
|
|
"adam_stats/m_t_max": 0.004364269785583019,
|
|
"adam_stats/m_t_mean": -3.4422544675782873e-11,
|
|
"adam_stats/m_t_min": -0.003123733215034008,
|
|
"adam_stats/v_t_max": 2.3226508346851915e-05,
|
|
"adam_stats/v_t_mean": 1.5332153949221272e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.703125,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.20983311533927917,
|
|
"all_logprobs": -0.011873790062963963,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.75,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.01732758805155754,
|
|
"answer_logprob_reward": 3.01825213432312,
|
|
"answer_logprob_reward/correct": 3.1318440437316895,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.030117960646748543,
|
|
"answer_logprob_reward/incorrect": 2.7492189407348633,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.32423263788223267,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.1469128578901291,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 576.8854370117188,
|
|
"completion_length/correct": 482.5555419921875,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 436.0,
|
|
"completion_length/correct/min": 173.0,
|
|
"completion_length/correct/p25": 307.5,
|
|
"completion_length/correct/p75": 634.5,
|
|
"completion_length/correct/var": 41892.47265625,
|
|
"completion_length/incorrect": 800.2982788085938,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 901.0,
|
|
"completion_length/incorrect/min": 206.0,
|
|
"completion_length/incorrect/p25": 574.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 66785.7890625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 544.0,
|
|
"completion_length/min": 173.0,
|
|
"completion_length/p25": 349.25,
|
|
"completion_length/p75": 785.0,
|
|
"completion_length/var": 70156.5390625,
|
|
"epoch": 0.5063291139240507,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2194030.0,
|
|
"grad_norm": 0.19211068749427795,
|
|
"kl": 0.011749267578125,
|
|
"learning_rate": 5.671166329088278e-06,
|
|
"len_reward": 0.08219938725233078,
|
|
"len_reward/correct": 0.19225099682807922,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.24640198051929474,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.04950905218720436,
|
|
"len_reward/correct/p75": 0.37484481930732727,
|
|
"len_reward/correct/var": 0.05272315815091133,
|
|
"len_reward/incorrect": -0.17844919860363007,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.13193117082118988,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.293021023273468,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.033872731029987335,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.0772036463022232,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.08989867568016052,
|
|
"len_reward/p75": 0.3173549175262451,
|
|
"len_reward/var": 0.075755275785923,
|
|
"loss": -0.7027,
|
|
"mean_logprobs": -0.01153564453125,
|
|
"mean_logprobs/var": 5.7220458984375e-05,
|
|
"missing_response_penalty": -0.109375,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.3684210479259491,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2368421107530594,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.09792213141918182,
|
|
"non_smoothed_logprobs/min": -17.1875,
|
|
"num_completions/total": 3840,
|
|
"policy_loss": -0.7026552557945251,
|
|
"policy_loss/max": 0.0008396075572818518,
|
|
"policy_loss/median": -0.9994357228279114,
|
|
"policy_loss/min": -0.99984210729599,
|
|
"policy_loss/p25": -0.9995936155319214,
|
|
"policy_loss/p75": 0.0003041678573936224,
|
|
"policy_loss/var": 0.20981955528259277,
|
|
"pred_value_log_probs": -0.6398011445999146,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.68359375,
|
|
"pred_value_log_probs/min": -1.3125,
|
|
"pred_value_log_probs/p25": -0.68359375,
|
|
"pred_value_log_probs/p75": -0.68359375,
|
|
"pred_value_log_probs/var": 0.04309263452887535,
|
|
"reward": 0.703125,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.20983311533927917,
|
|
"rewards/accuracy_reward": 0.703125,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.20983311533927917,
|
|
"rewards/answer_logprob_reward": 3.01825213432312,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.1469128578901291,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.08219938725233078,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.0772036463022232,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.08989867568016052,
|
|
"rewards/len_reward/p75": 0.3173549175262451,
|
|
"rewards/len_reward/var": 0.075755275785923,
|
|
"rewards/missing_response_penalty": -0.109375,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.09792213141918182,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 20,
|
|
"value_loss": 3.015625,
|
|
"values": -0.8254277110099792,
|
|
"values/max": 2.000021457672119,
|
|
"values/median": -0.8076171875,
|
|
"values/min": -1.94500732421875,
|
|
"values/p25": -0.8076171875,
|
|
"values/p75": -0.8076171875,
|
|
"values/var": 0.1680065244436264
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 3.718593143275939e-05,
|
|
"adam_stats/lr_effective_mean": 1.5384329921097617e-10,
|
|
"adam_stats/lr_effective_min": -3.6357450881041586e-05,
|
|
"adam_stats/m_t_max": 0.004364269785583019,
|
|
"adam_stats/m_t_mean": -3.4422544675782873e-11,
|
|
"adam_stats/m_t_min": -0.003123733215034008,
|
|
"adam_stats/v_t_max": 2.3226508346851915e-05,
|
|
"adam_stats/v_t_mean": 1.5332153949221272e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.5063291139240507,
|
|
"eval_accuracy_reward": 0.6566666960716248,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2262096107006073,
|
|
"eval_advantages": 0.6566666960716248,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2262096107006073,
|
|
"eval_all_logprobs": -0.013537714257836342,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.4375,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02079758048057556,
|
|
"eval_answer_logprob_reward": 3.0227293968200684,
|
|
"eval_answer_logprob_reward/correct": 3.1032936573028564,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.0898236483335495,
|
|
"eval_answer_logprob_reward/incorrect": 2.8686399459838867,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.3311939537525177,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.18431931734085083,
|
|
"eval_completion_length": 537.4733276367188,
|
|
"eval_completion_length/correct": 474.22332763671875,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 448.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 304.0,
|
|
"eval_completion_length/correct/p75": 600.0,
|
|
"eval_completion_length/correct/var": 44748.88671875,
|
|
"eval_completion_length/incorrect": 658.4465942382812,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 591.0,
|
|
"eval_completion_length/incorrect/min": 167.0,
|
|
"eval_completion_length/incorrect/p25": 411.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 84326.3359375,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 481.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 327.0,
|
|
"eval_completion_length/p75": 702.0,
|
|
"eval_completion_length/var": 65777.703125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 3448093.0,
|
|
"eval_len_reward": 0.04659387096762657,
|
|
"eval_len_reward/correct": 0.14398375153541565,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.18306010961532593,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.012527751736342907,
|
|
"eval_len_reward/correct/p75": 0.2897033095359802,
|
|
"eval_len_reward/correct/var": 0.053421687334775925,
|
|
"eval_len_reward/incorrect": -0.1396760791540146,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.024859286844730377,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2602172791957855,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.033676546066999435,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.08028224855661392,
|
|
"eval_len_reward/p75": 0.2392531782388687,
|
|
"eval_len_reward/var": 0.06470870971679688,
|
|
"eval_mean_logprobs": -0.01361083984375,
|
|
"eval_mean_logprobs/var": 0.00010442733764648438,
|
|
"eval_missing_response_penalty": -0.0833333358168602,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.24271844327449799,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.18560825288295746,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_num_completions/total": 6000,
|
|
"eval_pred_value_log_probs": -0.63873291015625,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6875,
|
|
"eval_pred_value_log_probs/min": -0.97265625,
|
|
"eval_pred_value_log_probs/p25": -0.6875,
|
|
"eval_pred_value_log_probs/p75": -0.6875,
|
|
"eval_pred_value_log_probs/var": 0.027704287320375443,
|
|
"eval_reward": 0.6566666960716248,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2262096107006073,
|
|
"eval_rewards/accuracy_reward": 0.6566666960716248,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2262096107006073,
|
|
"eval_rewards/answer_logprob_reward": 3.0227293968200684,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.18431931734085083,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.04659387096762657,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.08028224855661392,
|
|
"eval_rewards/len_reward/p75": 0.2392531782388687,
|
|
"eval_rewards/len_reward/var": 0.06470870971679688,
|
|
"eval_rewards/missing_response_penalty": -0.0833333358168602,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_runtime": 178.2592,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8452271819114685,
|
|
"eval_values/max": 1.9911384582519531,
|
|
"eval_values/median": -0.8037109375,
|
|
"eval_values/min": -1.9876670837402344,
|
|
"eval_values/p25": -0.8037109375,
|
|
"eval_values/p75": -0.8037109375,
|
|
"eval_values/var": 0.1308903843164444,
|
|
"step": 20
|
|
},
|
|
{
|
|
"accuracy_reward": 0.734375,
|
|
"accuracy_reward/correct": 0.9999999403953552,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.19608965516090393,
|
|
"adam_stats/lr_effective_max": 3.338371971040033e-05,
|
|
"adam_stats/lr_effective_mean": 1.3322469516463542e-10,
|
|
"adam_stats/lr_effective_min": -3.350756742293015e-05,
|
|
"adam_stats/m_t_max": 0.0037767805624753237,
|
|
"adam_stats/m_t_mean": -3.4601509157905497e-11,
|
|
"adam_stats/m_t_min": -0.002743084216490388,
|
|
"adam_stats/v_t_max": 2.3205564502859488e-05,
|
|
"adam_stats/v_t_mean": 1.5373737439344781e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.734375,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.19608965516090393,
|
|
"all_logprobs": -0.013131962157785892,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -12.25,
|
|
"all_logprobs/p25": -1.1920928955078125e-07,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.021637005731463432,
|
|
"answer_logprob_reward": 3.0625293254852295,
|
|
"answer_logprob_reward/correct": 3.1485581398010254,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.019714640453457832,
|
|
"answer_logprob_reward/incorrect": 2.8246846199035645,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.2762133777141571,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.10732635110616684,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 531.1614990234375,
|
|
"completion_length/correct": 446.39715576171875,
|
|
"completion_length/correct/max": 1020.0,
|
|
"completion_length/correct/median": 412.0,
|
|
"completion_length/correct/min": 135.0,
|
|
"completion_length/correct/p25": 285.0,
|
|
"completion_length/correct/p75": 579.0,
|
|
"completion_length/correct/var": 38654.16796875,
|
|
"completion_length/incorrect": 765.5098266601562,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 888.0,
|
|
"completion_length/incorrect/min": 280.0,
|
|
"completion_length/incorrect/p25": 501.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 76984.9765625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 462.0,
|
|
"completion_length/min": 135.0,
|
|
"completion_length/p25": 320.25,
|
|
"completion_length/p75": 666.75,
|
|
"completion_length/var": 68454.40625,
|
|
"epoch": 0.5316455696202531,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2296013.0,
|
|
"grad_norm": 0.20857343077659607,
|
|
"kl": 0.0126953125,
|
|
"learning_rate": 5.224324151752575e-06,
|
|
"len_reward": 0.09813292324542999,
|
|
"len_reward/correct": 0.20786333084106445,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2633753716945648,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.052776601165533066,
|
|
"len_reward/correct/p75": 0.363953173160553,
|
|
"len_reward/correct/var": 0.04442443698644638,
|
|
"len_reward/incorrect": -0.2052394300699234,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1717289686203003,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3497586250305176,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03765484690666199,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.13923776149749756,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.07858544588088989,
|
|
"len_reward/p75": 0.32227522134780884,
|
|
"len_reward/var": 0.0758831650018692,
|
|
"loss": -0.7339,
|
|
"mean_logprobs": -0.01287841796875,
|
|
"mean_logprobs/var": 0.00011491775512695312,
|
|
"missing_response_penalty": -0.09375,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.3529411852359772,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2329411655664444,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.08540575951337814,
|
|
"non_smoothed_logprobs/min": -17.4375,
|
|
"num_completions/total": 4032,
|
|
"policy_loss": -0.7338664531707764,
|
|
"policy_loss/max": 0.000989806023426354,
|
|
"policy_loss/median": -0.9994437098503113,
|
|
"policy_loss/min": -0.9997986555099487,
|
|
"policy_loss/p25": -0.9995540380477905,
|
|
"policy_loss/p75": 0.0002309512346982956,
|
|
"policy_loss/var": 0.19607244431972504,
|
|
"pred_value_log_probs": -0.6659946441650391,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6875,
|
|
"pred_value_log_probs/min": -1.2265625,
|
|
"pred_value_log_probs/p25": -0.6875,
|
|
"pred_value_log_probs/p75": -0.6875,
|
|
"pred_value_log_probs/var": 0.025847474113106728,
|
|
"reward": 0.734375,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.19608965516090393,
|
|
"rewards/accuracy_reward": 0.734375,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.19608965516090393,
|
|
"rewards/answer_logprob_reward": 3.0625293254852295,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.10732635110616684,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09813292324542999,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.13923776149749756,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.07858544588088989,
|
|
"rewards/len_reward/p75": 0.32227522134780884,
|
|
"rewards/len_reward/var": 0.0758831650018692,
|
|
"rewards/missing_response_penalty": -0.09375,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.08540575951337814,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 21,
|
|
"value_loss": 2.7421875,
|
|
"values": -0.7915716171264648,
|
|
"values/max": 1.9507980346679688,
|
|
"values/median": -0.8037109375,
|
|
"values/min": -1.9878120422363281,
|
|
"values/p25": -0.8037109375,
|
|
"values/p75": -0.8037109375,
|
|
"values/var": 0.12283516675233841
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 3.338371971040033e-05,
|
|
"adam_stats/lr_effective_mean": 1.3322469516463542e-10,
|
|
"adam_stats/lr_effective_min": -3.350756742293015e-05,
|
|
"adam_stats/m_t_max": 0.0037767805624753237,
|
|
"adam_stats/m_t_mean": -3.4601509157905497e-11,
|
|
"adam_stats/m_t_min": -0.002743084216490388,
|
|
"adam_stats/v_t_max": 2.3205564502859488e-05,
|
|
"adam_stats/v_t_mean": 1.5373737439344781e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.5316455696202531,
|
|
"eval_accuracy_reward": 0.6633333563804626,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22406914830207825,
|
|
"eval_advantages": 0.6633333563804626,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22406914830207825,
|
|
"eval_all_logprobs": -0.014047027565538883,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -10.4375,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.023722998797893524,
|
|
"eval_answer_logprob_reward": 3.0409231185913086,
|
|
"eval_answer_logprob_reward/correct": 3.118717908859253,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.0664416253566742,
|
|
"eval_answer_logprob_reward/incorrect": 2.8876430988311768,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.27647578716278076,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.14842931926250458,
|
|
"eval_completion_length": 549.856689453125,
|
|
"eval_completion_length/correct": 477.6783752441406,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 448.0,
|
|
"eval_completion_length/correct/min": 114.0,
|
|
"eval_completion_length/correct/p25": 300.5,
|
|
"eval_completion_length/correct/p75": 617.5,
|
|
"eval_completion_length/correct/var": 49479.5703125,
|
|
"eval_completion_length/incorrect": 692.0692749023438,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 661.0,
|
|
"eval_completion_length/incorrect/min": 180.0,
|
|
"eval_completion_length/incorrect/p25": 440.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 83685.796875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 504.0,
|
|
"eval_completion_length/min": 114.0,
|
|
"eval_completion_length/p25": 321.75,
|
|
"eval_completion_length/p75": 742.75,
|
|
"eval_completion_length/var": 71053.2890625,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 3613050.0,
|
|
"eval_len_reward": 0.05123516172170639,
|
|
"eval_len_reward/correct": 0.15512846410274506,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.19577352702617645,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.010729953646659851,
|
|
"eval_len_reward/correct/p75": 0.3160973787307739,
|
|
"eval_len_reward/correct/var": 0.054460883140563965,
|
|
"eval_len_reward/incorrect": -0.15346552431583405,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.07097288966178894,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.27089521288871765,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03267636150121689,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.008183632977306843,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.09504184871912003,
|
|
"eval_len_reward/p75": 0.2602965235710144,
|
|
"eval_len_reward/var": 0.06833110004663467,
|
|
"eval_mean_logprobs": -0.0135498046875,
|
|
"eval_mean_logprobs/var": 0.000118255615234375,
|
|
"eval_missing_response_penalty": -0.10000000149011612,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2970297038555145,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.21089109778404236,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.09030100703239441,
|
|
"eval_num_completions/total": 6300,
|
|
"eval_pred_value_log_probs": -0.6350853443145752,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.68359375,
|
|
"eval_pred_value_log_probs/min": -0.98046875,
|
|
"eval_pred_value_log_probs/p25": -0.68359375,
|
|
"eval_pred_value_log_probs/p75": -0.68359375,
|
|
"eval_pred_value_log_probs/var": 0.027328863739967346,
|
|
"eval_reward": 0.6633333563804626,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22406914830207825,
|
|
"eval_rewards/accuracy_reward": 0.6633333563804626,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22406914830207825,
|
|
"eval_rewards/answer_logprob_reward": 3.0409231185913086,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.14842931926250458,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.05123516172170639,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.008183632977306843,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.09504184871912003,
|
|
"eval_rewards/len_reward/p75": 0.2602965235710144,
|
|
"eval_rewards/len_reward/var": 0.06833110004663467,
|
|
"eval_rewards/missing_response_penalty": -0.10000000149011612,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.09030100703239441,
|
|
"eval_runtime": 175.1157,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8495923280715942,
|
|
"eval_values/max": 1.9913177490234375,
|
|
"eval_values/median": -0.80859375,
|
|
"eval_values/min": -1.9885406494140625,
|
|
"eval_values/p25": -0.80859375,
|
|
"eval_values/p75": -0.80859375,
|
|
"eval_values/var": 0.12950313091278076,
|
|
"step": 21
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7604166865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.18313699960708618,
|
|
"adam_stats/lr_effective_max": 3.071120227104984e-05,
|
|
"adam_stats/lr_effective_mean": 5.2620817425630406e-11,
|
|
"adam_stats/lr_effective_min": -3.083813135162927e-05,
|
|
"adam_stats/m_t_max": 0.003942315466701984,
|
|
"adam_stats/m_t_mean": -2.766375670204546e-11,
|
|
"adam_stats/m_t_min": -0.002786028664559126,
|
|
"adam_stats/v_t_max": 2.3211867301142775e-05,
|
|
"adam_stats/v_t_mean": 1.5432596606884674e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7604166865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.18313699960708618,
|
|
"all_logprobs": -0.012917487882077694,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.75,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.018993373960256577,
|
|
"answer_logprob_reward": 3.0326926708221436,
|
|
"answer_logprob_reward/correct": 3.098508596420288,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.05836192145943642,
|
|
"answer_logprob_reward/incorrect": 2.823798894882202,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.2675291895866394,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.12115713208913803,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 491.4895935058594,
|
|
"completion_length/correct": 417.28765869140625,
|
|
"completion_length/correct/max": 1021.0,
|
|
"completion_length/correct/median": 386.0,
|
|
"completion_length/correct/min": 138.0,
|
|
"completion_length/correct/p25": 299.25,
|
|
"completion_length/correct/p75": 500.5,
|
|
"completion_length/correct/var": 28962.716796875,
|
|
"completion_length/incorrect": 727.0,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 738.0,
|
|
"completion_length/incorrect/min": 217.0,
|
|
"completion_length/incorrect/p25": 458.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 83329.6875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 425.0,
|
|
"completion_length/min": 138.0,
|
|
"completion_length/p25": 316.0,
|
|
"completion_length/p75": 594.5,
|
|
"completion_length/var": 59186.8671875,
|
|
"epoch": 0.5569620253164557,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2390379.0,
|
|
"grad_norm": 0.22681620717048645,
|
|
"kl": 0.01324462890625,
|
|
"learning_rate": 4.775675848247427e-06,
|
|
"len_reward": 0.09454120695590973,
|
|
"len_reward/correct": 0.18561342358589172,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2097862809896469,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.03180480748414993,
|
|
"len_reward/correct/p75": 0.36853471398353577,
|
|
"len_reward/correct/var": 0.04988931864500046,
|
|
"len_reward/incorrect": -0.1945141702890396,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.12032489478588104,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.43185335397720337,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.04208043962717056,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.08360301703214645,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.0493905171751976,
|
|
"len_reward/p75": 0.31959444284439087,
|
|
"len_reward/var": 0.07425107806921005,
|
|
"loss": -0.7599,
|
|
"mean_logprobs": -0.01251220703125,
|
|
"mean_logprobs/var": 7.963180541992188e-05,
|
|
"missing_response_penalty": -0.0885416716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.3695652186870575,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2381642609834671,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.08112456649541855,
|
|
"non_smoothed_logprobs/min": -17.5625,
|
|
"num_completions/total": 4224,
|
|
"policy_loss": -0.759887158870697,
|
|
"policy_loss/max": 0.0012284929398447275,
|
|
"policy_loss/median": -0.9994112253189087,
|
|
"policy_loss/min": -0.9997952580451965,
|
|
"policy_loss/p25": -0.9995701313018799,
|
|
"policy_loss/p75": -0.9989782571792603,
|
|
"policy_loss/var": 0.18312634527683258,
|
|
"pred_value_log_probs": -0.6359685659408569,
|
|
"pred_value_log_probs/max": -0.00982666015625,
|
|
"pred_value_log_probs/median": -0.68359375,
|
|
"pred_value_log_probs/min": -1.0625,
|
|
"pred_value_log_probs/p25": -0.68359375,
|
|
"pred_value_log_probs/p75": -0.68359375,
|
|
"pred_value_log_probs/var": 0.028932176530361176,
|
|
"reward": 0.7604166865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.18313699960708618,
|
|
"rewards/accuracy_reward": 0.7604166865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.18313699960708618,
|
|
"rewards/answer_logprob_reward": 3.0326926708221436,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.12115713208913803,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09454120695590973,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.08360301703214645,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.0493905171751976,
|
|
"rewards/len_reward/p75": 0.31959444284439087,
|
|
"rewards/len_reward/var": 0.07425107806921005,
|
|
"rewards/missing_response_penalty": -0.0885416716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.08112456649541855,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 22,
|
|
"value_loss": 2.7421875,
|
|
"values": -0.7965652346611023,
|
|
"values/max": 1.78765869140625,
|
|
"values/median": -0.80859375,
|
|
"values/min": -1.9856138229370117,
|
|
"values/p25": -0.80859375,
|
|
"values/p75": -0.80859375,
|
|
"values/var": 0.180428147315979
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 3.071120227104984e-05,
|
|
"adam_stats/lr_effective_mean": 5.2620817425630406e-11,
|
|
"adam_stats/lr_effective_min": -3.083813135162927e-05,
|
|
"adam_stats/m_t_max": 0.003942315466701984,
|
|
"adam_stats/m_t_mean": -2.766375670204546e-11,
|
|
"adam_stats/m_t_min": -0.002786028664559126,
|
|
"adam_stats/v_t_max": 2.3211867301142775e-05,
|
|
"adam_stats/v_t_mean": 1.5432596606884674e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.5569620253164557,
|
|
"eval_accuracy_reward": 0.6333333253860474,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23299887776374817,
|
|
"eval_advantages": 0.6333333253860474,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23299887776374817,
|
|
"eval_all_logprobs": -0.01301631610840559,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -8.125,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.01946735754609108,
|
|
"eval_answer_logprob_reward": 3.0356905460357666,
|
|
"eval_answer_logprob_reward/correct": 3.106058120727539,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07760242372751236,
|
|
"eval_answer_logprob_reward/incorrect": 2.914146661758423,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.22009778022766113,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.137870654463768,
|
|
"eval_completion_length": 538.27001953125,
|
|
"eval_completion_length/correct": 465.2684326171875,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 454.0,
|
|
"eval_completion_length/correct/min": 119.0,
|
|
"eval_completion_length/correct/p25": 311.25,
|
|
"eval_completion_length/correct/p75": 597.75,
|
|
"eval_completion_length/correct/var": 40518.1484375,
|
|
"eval_completion_length/incorrect": 664.3635864257812,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 619.0,
|
|
"eval_completion_length/incorrect/min": 168.0,
|
|
"eval_completion_length/incorrect/p25": 422.5,
|
|
"eval_completion_length/incorrect/p75": 1009.5,
|
|
"eval_completion_length/incorrect/var": 80659.25,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 484.0,
|
|
"eval_completion_length/min": 119.0,
|
|
"eval_completion_length/p25": 331.25,
|
|
"eval_completion_length/p75": 686.0,
|
|
"eval_completion_length/var": 64251.83203125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 3774531.0,
|
|
"eval_len_reward": 0.03958354517817497,
|
|
"eval_len_reward/correct": 0.15136002004146576,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.17757193744182587,
|
|
"eval_len_reward/correct/min": -0.46847546100616455,
|
|
"eval_len_reward/correct/p25": 0.008404039777815342,
|
|
"eval_len_reward/correct/p75": 0.32655590772628784,
|
|
"eval_len_reward/correct/var": 0.05165595933794975,
|
|
"eval_len_reward/incorrect": -0.15348489582538605,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.060496795922517776,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.28577178716659546,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03317929431796074,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.12148172408342361,
|
|
"eval_len_reward/p75": 0.23447690904140472,
|
|
"eval_len_reward/var": 0.06640024483203888,
|
|
"eval_mean_logprobs": -0.01336669921875,
|
|
"eval_mean_logprobs/var": 9.918212890625e-05,
|
|
"eval_missing_response_penalty": -0.08666666597127914,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.23636363446712494,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.18215179443359375,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07942028343677521,
|
|
"eval_num_completions/total": 6600,
|
|
"eval_pred_value_log_probs": -0.6274722814559937,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.67578125,
|
|
"eval_pred_value_log_probs/min": -0.99609375,
|
|
"eval_pred_value_log_probs/p25": -0.67578125,
|
|
"eval_pred_value_log_probs/p75": -0.67578125,
|
|
"eval_pred_value_log_probs/var": 0.02641473338007927,
|
|
"eval_reward": 0.6333333253860474,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23299887776374817,
|
|
"eval_rewards/accuracy_reward": 0.6333333253860474,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23299887776374817,
|
|
"eval_rewards/answer_logprob_reward": 3.0356905460357666,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.137870654463768,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.03958354517817497,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.12148172408342361,
|
|
"eval_rewards/len_reward/p75": 0.23447690904140472,
|
|
"eval_rewards/len_reward/var": 0.06640024483203888,
|
|
"eval_rewards/missing_response_penalty": -0.08666666597127914,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07942028343677521,
|
|
"eval_runtime": 172.5601,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8610339760780334,
|
|
"eval_values/max": 1.9912223815917969,
|
|
"eval_values/median": -0.8212890625,
|
|
"eval_values/min": -1.9895038604736328,
|
|
"eval_values/p25": -0.8212890625,
|
|
"eval_values/p75": -0.8212890625,
|
|
"eval_values/var": 0.1292973905801773,
|
|
"step": 22
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7395833730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.19360819458961487,
|
|
"adam_stats/lr_effective_max": 2.803716233756859e-05,
|
|
"adam_stats/lr_effective_mean": 1.5093860883386157e-10,
|
|
"adam_stats/lr_effective_min": -2.8081658456358127e-05,
|
|
"adam_stats/m_t_max": 0.0031940797343850136,
|
|
"adam_stats/m_t_mean": -1.9256929384425803e-11,
|
|
"adam_stats/m_t_min": -0.0024208729155361652,
|
|
"adam_stats/v_t_max": 2.3201186195365153e-05,
|
|
"adam_stats/v_t_mean": 1.5473259609363743e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7395833730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.19360819458961487,
|
|
"all_logprobs": -0.01277267187833786,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -12.75,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.020812146365642548,
|
|
"answer_logprob_reward": 2.9443728923797607,
|
|
"answer_logprob_reward/correct": 3.101149082183838,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.057205647230148315,
|
|
"answer_logprob_reward/incorrect": 2.4991276264190674,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 0.8754687309265137,
|
|
"answer_logprob_reward/incorrect/p25": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.8718653917312622,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 0.8754687309265137,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.33607202768325806,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 554.6510620117188,
|
|
"completion_length/correct": 474.6056213378906,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 385.0,
|
|
"completion_length/correct/min": 155.0,
|
|
"completion_length/correct/p25": 311.5,
|
|
"completion_length/correct/p75": 595.25,
|
|
"completion_length/correct/var": 50056.6640625,
|
|
"completion_length/incorrect": 781.97998046875,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 827.0,
|
|
"completion_length/incorrect/min": 205.0,
|
|
"completion_length/incorrect/p25": 569.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 64115.1640625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 468.0,
|
|
"completion_length/min": 155.0,
|
|
"completion_length/p25": 344.0,
|
|
"completion_length/p75": 758.5,
|
|
"completion_length/var": 71693.1328125,
|
|
"epoch": 0.5822784810126582,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2496872.0,
|
|
"grad_norm": 0.21534405648708344,
|
|
"kl": 0.01239013671875,
|
|
"learning_rate": 4.3288336709117246e-06,
|
|
"len_reward": 0.09688381850719452,
|
|
"len_reward/correct": 0.19501151144504547,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.22932861745357513,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.04732845723628998,
|
|
"len_reward/correct/p75": 0.36181944608688354,
|
|
"len_reward/correct/var": 0.05219462141394615,
|
|
"len_reward/incorrect": -0.18179886043071747,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.14137931168079376,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3377705514431,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03440873324871063,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.11601041257381439,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.05947368964552879,
|
|
"len_reward/p75": 0.3168730139732361,
|
|
"len_reward/var": 0.07484813779592514,
|
|
"loss": -0.7391,
|
|
"mean_logprobs": -0.0126953125,
|
|
"mean_logprobs/var": 8.630752563476562e-05,
|
|
"missing_response_penalty": -0.1145833358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4399999976158142,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2514285743236542,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.10198517888784409,
|
|
"non_smoothed_logprobs/min": -17.3125,
|
|
"num_completions/total": 4416,
|
|
"policy_loss": -0.739088237285614,
|
|
"policy_loss/max": 0.0009990279795601964,
|
|
"policy_loss/median": -0.9994358420372009,
|
|
"policy_loss/min": -0.9997886419296265,
|
|
"policy_loss/p25": -0.9995859861373901,
|
|
"policy_loss/p75": 0.0001322152093052864,
|
|
"policy_loss/var": 0.19358791410923004,
|
|
"pred_value_log_probs": -0.6088201403617859,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.67578125,
|
|
"pred_value_log_probs/min": -1.234375,
|
|
"pred_value_log_probs/p25": -0.67578125,
|
|
"pred_value_log_probs/p75": -0.67578125,
|
|
"pred_value_log_probs/var": 0.03758978098630905,
|
|
"reward": 0.7395833730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.19360819458961487,
|
|
"rewards/accuracy_reward": 0.7395833730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.19360819458961487,
|
|
"rewards/answer_logprob_reward": 2.9443728923797607,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 0.8754687309265137,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.33607202768325806,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09688381850719452,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.11601041257381439,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.05947368964552879,
|
|
"rewards/len_reward/p75": 0.3168730139732361,
|
|
"rewards/len_reward/var": 0.07484813779592514,
|
|
"rewards/missing_response_penalty": -0.1145833358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.10198517888784409,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 23,
|
|
"value_loss": 2.96875,
|
|
"values": -0.8436825275421143,
|
|
"values/max": 1.195037841796875,
|
|
"values/median": -0.8212890625,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.8212890625,
|
|
"values/p75": -0.8212890625,
|
|
"values/var": 0.17299538850784302
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 2.803716233756859e-05,
|
|
"adam_stats/lr_effective_mean": 1.5093860883386157e-10,
|
|
"adam_stats/lr_effective_min": -2.8081658456358127e-05,
|
|
"adam_stats/m_t_max": 0.0031940797343850136,
|
|
"adam_stats/m_t_mean": -1.9256929384425803e-11,
|
|
"adam_stats/m_t_min": -0.0024208729155361652,
|
|
"adam_stats/v_t_max": 2.3201186195365153e-05,
|
|
"adam_stats/v_t_mean": 1.5473259609363743e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.5822784810126582,
|
|
"eval_accuracy_reward": 0.6366666555404663,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2320958524942398,
|
|
"eval_advantages": 0.6366666555404663,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2320958524942398,
|
|
"eval_all_logprobs": -0.013486981391906738,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -6.5,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.01943836733698845,
|
|
"eval_answer_logprob_reward": 3.047809362411499,
|
|
"eval_answer_logprob_reward/correct": 3.12178111076355,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06521151214838028,
|
|
"eval_answer_logprob_reward/incorrect": 2.9181885719299316,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.2903265655040741,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.15592624247074127,
|
|
"eval_completion_length": 526.106689453125,
|
|
"eval_completion_length/correct": 457.0052490234375,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 426.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 301.0,
|
|
"eval_completion_length/correct/p75": 587.5,
|
|
"eval_completion_length/correct/var": 41787.1015625,
|
|
"eval_completion_length/incorrect": 647.192626953125,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 592.0,
|
|
"eval_completion_length/incorrect/min": 178.0,
|
|
"eval_completion_length/incorrect/p25": 402.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 80823.3046875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 463.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 329.75,
|
|
"eval_completion_length/p75": 684.25,
|
|
"eval_completion_length/var": 64142.58203125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 3932363.0,
|
|
"eval_len_reward": 0.04952501505613327,
|
|
"eval_len_reward/correct": 0.15692557394504547,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.17270317673683167,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.0006625441601499915,
|
|
"eval_len_reward/correct/p75": 0.32966095209121704,
|
|
"eval_len_reward/correct/var": 0.051744624972343445,
|
|
"eval_len_reward/incorrect": -0.13867229223251343,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.014140008948743343,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2403801530599594,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03106965683400631,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.09050571173429489,
|
|
"eval_len_reward/p75": 0.2296256422996521,
|
|
"eval_len_reward/var": 0.06438378244638443,
|
|
"eval_mean_logprobs": -0.013671875,
|
|
"eval_mean_logprobs/var": 0.00010061264038085938,
|
|
"eval_missing_response_penalty": -0.09666667133569717,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.26605504751205444,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.19707782566547394,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0876142680644989,
|
|
"eval_num_completions/total": 6900,
|
|
"eval_pred_value_log_probs": -0.624318540096283,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.671875,
|
|
"eval_pred_value_log_probs/min": -0.984375,
|
|
"eval_pred_value_log_probs/p25": -0.671875,
|
|
"eval_pred_value_log_probs/p75": -0.671875,
|
|
"eval_pred_value_log_probs/var": 0.02601056918501854,
|
|
"eval_reward": 0.6366666555404663,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2320958524942398,
|
|
"eval_rewards/accuracy_reward": 0.6366666555404663,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2320958524942398,
|
|
"eval_rewards/answer_logprob_reward": 3.047809362411499,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.15592624247074127,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.04952501505613327,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.09050571173429489,
|
|
"eval_rewards/len_reward/p75": 0.2296256422996521,
|
|
"eval_rewards/len_reward/var": 0.06438378244638443,
|
|
"eval_rewards/missing_response_penalty": -0.09666667133569717,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0876142680644989,
|
|
"eval_runtime": 168.7882,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8640043139457703,
|
|
"eval_values/max": 1.9913520812988281,
|
|
"eval_values/median": -0.82568359375,
|
|
"eval_values/min": -1.9897537231445312,
|
|
"eval_values/p25": -0.82568359375,
|
|
"eval_values/p75": -0.82568359375,
|
|
"eval_values/var": 0.12934480607509613,
|
|
"step": 23
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7916666865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.16579407453536987,
|
|
"adam_stats/lr_effective_max": 2.521297210478224e-05,
|
|
"adam_stats/lr_effective_mean": 6.384962003558314e-11,
|
|
"adam_stats/lr_effective_min": -2.474541906849481e-05,
|
|
"adam_stats/m_t_max": 0.002732001943513751,
|
|
"adam_stats/m_t_mean": -1.6766574240101306e-11,
|
|
"adam_stats/m_t_min": -0.0020878624636679888,
|
|
"adam_stats/v_t_max": 2.3180020434665494e-05,
|
|
"adam_stats/v_t_mean": 1.5528077955406783e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7916666865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.16579407453536987,
|
|
"all_logprobs": -0.014185355976223946,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.84375,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.022149622440338135,
|
|
"answer_logprob_reward": 3.0299673080444336,
|
|
"answer_logprob_reward/correct": 3.1062088012695312,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.0537831075489521,
|
|
"answer_logprob_reward/incorrect": 2.740250587463379,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.316007137298584,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.1292486935853958,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 504.203125,
|
|
"completion_length/correct": 437.631591796875,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 388.0,
|
|
"completion_length/correct/min": 127.0,
|
|
"completion_length/correct/p25": 291.0,
|
|
"completion_length/correct/p75": 518.75,
|
|
"completion_length/correct/var": 45088.0546875,
|
|
"completion_length/incorrect": 757.1749877929688,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 835.0,
|
|
"completion_length/incorrect/min": 223.0,
|
|
"completion_length/incorrect/p25": 451.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 84737.8984375,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 422.0,
|
|
"completion_length/min": 127.0,
|
|
"completion_length/p25": 307.75,
|
|
"completion_length/p75": 661.5,
|
|
"completion_length/var": 69876.921875,
|
|
"epoch": 0.6075949367088608,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2593679.0,
|
|
"grad_norm": 0.23429417610168457,
|
|
"kl": 0.013427734375,
|
|
"learning_rate": 3.887395330218429e-06,
|
|
"len_reward": 0.10179547965526581,
|
|
"len_reward/correct": 0.18434910476207733,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.22507065534591675,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.0629768818616867,
|
|
"len_reward/correct/p75": 0.3572998344898224,
|
|
"len_reward/correct/var": 0.055906061083078384,
|
|
"len_reward/incorrect": -0.21190838515758514,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.2201894074678421,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3721355199813843,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.037402331829071045,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.1524021029472351,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.05030941963195801,
|
|
"len_reward/p75": 0.3255058526992798,
|
|
"len_reward/var": 0.07786809653043747,
|
|
"loss": -0.7911,
|
|
"mean_logprobs": -0.0133056640625,
|
|
"mean_logprobs/var": 0.00011014938354492188,
|
|
"missing_response_penalty": -0.0833333358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4000000059604645,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.24615386128425598,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.07678883522748947,
|
|
"non_smoothed_logprobs/min": -17.25,
|
|
"num_completions/total": 4608,
|
|
"policy_loss": -0.7911297678947449,
|
|
"policy_loss/max": 0.0009242998785339296,
|
|
"policy_loss/median": -0.999406099319458,
|
|
"policy_loss/min": -0.9998186826705933,
|
|
"policy_loss/p25": -0.999532163143158,
|
|
"policy_loss/p75": -0.9991399645805359,
|
|
"policy_loss/var": 0.1657712310552597,
|
|
"pred_value_log_probs": -0.605526328086853,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.671875,
|
|
"pred_value_log_probs/min": -1.21875,
|
|
"pred_value_log_probs/p25": -0.671875,
|
|
"pred_value_log_probs/p75": -0.671875,
|
|
"pred_value_log_probs/var": 0.03997424617409706,
|
|
"reward": 0.7916666865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.16579407453536987,
|
|
"rewards/accuracy_reward": 0.7916666865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.16579407453536987,
|
|
"rewards/answer_logprob_reward": 3.0299673080444336,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.1292486935853958,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.10179547965526581,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.1524021029472351,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.05030941963195801,
|
|
"rewards/len_reward/p75": 0.3255058526992798,
|
|
"rewards/len_reward/var": 0.07786809653043747,
|
|
"rewards/missing_response_penalty": -0.0833333358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.07678883522748947,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 24,
|
|
"value_loss": 3.0546875,
|
|
"values": -0.873255729675293,
|
|
"values/max": 1.30712890625,
|
|
"values/median": -0.82568359375,
|
|
"values/min": -1.9909396171569824,
|
|
"values/p25": -0.82568359375,
|
|
"values/p75": -0.82568359375,
|
|
"values/var": 0.20940503478050232
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 2.521297210478224e-05,
|
|
"adam_stats/lr_effective_mean": 6.384962003558314e-11,
|
|
"adam_stats/lr_effective_min": -2.474541906849481e-05,
|
|
"adam_stats/m_t_max": 0.002732001943513751,
|
|
"adam_stats/m_t_mean": -1.6766574240101306e-11,
|
|
"adam_stats/m_t_min": -0.0020878624636679888,
|
|
"adam_stats/v_t_max": 2.3180020434665494e-05,
|
|
"adam_stats/v_t_mean": 1.5528077955406783e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.6075949367088608,
|
|
"eval_accuracy_reward": 0.6466667056083679,
|
|
"eval_accuracy_reward/correct": 0.9999999403953552,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22925306856632233,
|
|
"eval_advantages": 0.6466667056083679,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22925306856632233,
|
|
"eval_all_logprobs": -0.01458882074803114,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -8.75,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.023142628371715546,
|
|
"eval_answer_logprob_reward": 3.075014114379883,
|
|
"eval_answer_logprob_reward/correct": 3.126248359680176,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06745291501283646,
|
|
"eval_answer_logprob_reward/incorrect": 2.9812450408935547,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/incorrect/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.19945694506168365,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.11840354651212692,
|
|
"eval_completion_length": 535.1699829101562,
|
|
"eval_completion_length/correct": 472.83502197265625,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 435.0,
|
|
"eval_completion_length/correct/min": 118.0,
|
|
"eval_completion_length/correct/p25": 302.0,
|
|
"eval_completion_length/correct/p75": 603.75,
|
|
"eval_completion_length/correct/var": 48038.92578125,
|
|
"eval_completion_length/incorrect": 649.2546997070312,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 628.0,
|
|
"eval_completion_length/incorrect/min": 170.0,
|
|
"eval_completion_length/incorrect/p25": 416.5,
|
|
"eval_completion_length/incorrect/p75": 942.25,
|
|
"eval_completion_length/incorrect/var": 77611.6640625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 483.0,
|
|
"eval_completion_length/min": 118.0,
|
|
"eval_completion_length/p25": 321.0,
|
|
"eval_completion_length/p75": 684.5,
|
|
"eval_completion_length/var": 65398.58203125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 4092914.0,
|
|
"eval_len_reward": 0.061788979917764664,
|
|
"eval_len_reward/correct": 0.15681885182857513,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1812652051448822,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.0326542966067791,
|
|
"eval_len_reward/correct/p75": 0.34234336018562317,
|
|
"eval_len_reward/correct/var": 0.05734947696328163,
|
|
"eval_len_reward/incorrect": -0.11213362216949463,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": 0.0,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.15305142104625702,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.028778472915291786,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.005759736523032188,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.0507662259042263,
|
|
"eval_len_reward/p75": 0.2570588290691376,
|
|
"eval_len_reward/var": 0.06370748579502106,
|
|
"eval_mean_logprobs": -0.013671875,
|
|
"eval_mean_logprobs/var": 0.0004291534423828125,
|
|
"eval_missing_response_penalty": -0.07999999821186066,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.22641509771347046,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.17681941390037537,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07384615391492844,
|
|
"eval_num_completions/total": 7200,
|
|
"eval_pred_value_log_probs": -0.620497465133667,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.66796875,
|
|
"eval_pred_value_log_probs/min": -1.0234375,
|
|
"eval_pred_value_log_probs/p25": -0.66796875,
|
|
"eval_pred_value_log_probs/p75": -0.66796875,
|
|
"eval_pred_value_log_probs/var": 0.026062848046422005,
|
|
"eval_reward": 0.6466667056083679,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22925306856632233,
|
|
"eval_rewards/accuracy_reward": 0.6466667056083679,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22925306856632233,
|
|
"eval_rewards/answer_logprob_reward": 3.075014114379883,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.11840354651212692,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.061788979917764664,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.005759736523032188,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.0507662259042263,
|
|
"eval_rewards/len_reward/p75": 0.2570588290691376,
|
|
"eval_rewards/len_reward/var": 0.06370748579502106,
|
|
"eval_rewards/missing_response_penalty": -0.07999999821186066,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07384615391492844,
|
|
"eval_runtime": 165.1828,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8670269250869751,
|
|
"eval_values/max": 1.991119384765625,
|
|
"eval_values/median": -0.82958984375,
|
|
"eval_values/min": -1.997873306274414,
|
|
"eval_values/p25": -0.82958984375,
|
|
"eval_values/p75": -0.82958984375,
|
|
"eval_values/var": 0.12986624240875244,
|
|
"step": 24
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7864583730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.16882090270519257,
|
|
"adam_stats/lr_effective_max": 2.1769656086689793e-05,
|
|
"adam_stats/lr_effective_mean": 5.913265710422166e-11,
|
|
"adam_stats/lr_effective_min": -2.242199843749404e-05,
|
|
"adam_stats/m_t_max": 0.0026235966943204403,
|
|
"adam_stats/m_t_mean": -2.350836886266805e-11,
|
|
"adam_stats/m_t_min": -0.0016450312687084079,
|
|
"adam_stats/v_t_max": 2.3159556803875603e-05,
|
|
"adam_stats/v_t_mean": 1.558106725238484e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7864583730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.16882090270519257,
|
|
"all_logprobs": -0.013827561400830746,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.3125,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.022680440917611122,
|
|
"answer_logprob_reward": 3.028197765350342,
|
|
"answer_logprob_reward/correct": 3.0954270362854004,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.05078166723251343,
|
|
"answer_logprob_reward/incorrect": 2.7805967330932617,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.2833702266216278,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.11595863103866577,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 514.9739990234375,
|
|
"completion_length/correct": 456.0331115722656,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 396.0,
|
|
"completion_length/correct/min": 120.0,
|
|
"completion_length/correct/p25": 304.5,
|
|
"completion_length/correct/p75": 590.0,
|
|
"completion_length/correct/var": 43072.3671875,
|
|
"completion_length/incorrect": 732.0487670898438,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 735.0,
|
|
"completion_length/incorrect/min": 254.0,
|
|
"completion_length/incorrect/p25": 435.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 86421.203125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 435.0,
|
|
"completion_length/min": 120.0,
|
|
"completion_length/p25": 317.75,
|
|
"completion_length/p75": 653.0,
|
|
"completion_length/var": 64786.703125,
|
|
"epoch": 0.6329113924050633,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2692554.0,
|
|
"grad_norm": 0.22061339020729065,
|
|
"kl": 0.01324462890625,
|
|
"learning_rate": 3.4549150281252635e-06,
|
|
"len_reward": 0.1224907785654068,
|
|
"len_reward/correct": 0.20522892475128174,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2400950938463211,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.07591985166072845,
|
|
"len_reward/correct/p75": 0.36015838384628296,
|
|
"len_reward/correct/var": 0.04885726049542427,
|
|
"len_reward/incorrect": -0.18222780525684357,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1176169365644455,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3276824951171875,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.035905227065086365,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.17908082902431488,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.03324875980615616,
|
|
"len_reward/p75": 0.34264105558395386,
|
|
"len_reward/var": 0.07123284786939621,
|
|
"loss": -0.7859,
|
|
"mean_logprobs": -0.0135498046875,
|
|
"mean_logprobs/var": 9.870529174804688e-05,
|
|
"missing_response_penalty": -0.0833333358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.39024388790130615,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.24390244483947754,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.07678883522748947,
|
|
"non_smoothed_logprobs/min": -17.25,
|
|
"num_completions/total": 4800,
|
|
"policy_loss": -0.7859289646148682,
|
|
"policy_loss/max": 0.0010629133321344852,
|
|
"policy_loss/median": -0.9994313716888428,
|
|
"policy_loss/min": -0.9997806549072266,
|
|
"policy_loss/p25": -0.9995715618133545,
|
|
"policy_loss/p75": -0.9991579055786133,
|
|
"policy_loss/var": 0.1688242405653,
|
|
"pred_value_log_probs": -0.6209506988525391,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.66796875,
|
|
"pred_value_log_probs/min": -1.1328125,
|
|
"pred_value_log_probs/p25": -0.66796875,
|
|
"pred_value_log_probs/p75": -0.66796875,
|
|
"pred_value_log_probs/var": 0.032860416918992996,
|
|
"reward": 0.7864583730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.16882090270519257,
|
|
"rewards/accuracy_reward": 0.7864583730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.16882090270519257,
|
|
"rewards/answer_logprob_reward": 3.028197765350342,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.11595863103866577,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.1224907785654068,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.17908082902431488,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.03324875980615616,
|
|
"rewards/len_reward/p75": 0.34264105558395386,
|
|
"rewards/len_reward/var": 0.07123284786939621,
|
|
"rewards/missing_response_penalty": -0.0833333358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.07678883522748947,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 25,
|
|
"value_loss": 3.015625,
|
|
"values": -0.8420132994651794,
|
|
"values/max": 1.9203028678894043,
|
|
"values/median": -0.82958984375,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.82958984375,
|
|
"values/p75": -0.82958984375,
|
|
"values/var": 0.1504109799861908
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 2.1769656086689793e-05,
|
|
"adam_stats/lr_effective_mean": 5.913265710422166e-11,
|
|
"adam_stats/lr_effective_min": -2.242199843749404e-05,
|
|
"adam_stats/m_t_max": 0.0026235966943204403,
|
|
"adam_stats/m_t_mean": -2.350836886266805e-11,
|
|
"adam_stats/m_t_min": -0.0016450312687084079,
|
|
"adam_stats/v_t_max": 2.3159556803875603e-05,
|
|
"adam_stats/v_t_mean": 1.558106725238484e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.6329113924050633,
|
|
"eval_accuracy_reward": 0.643333375453949,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23022297024726868,
|
|
"eval_advantages": 0.643333375453949,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23022297024726868,
|
|
"eval_all_logprobs": -0.012885897420346737,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -6.25,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.01903092861175537,
|
|
"eval_answer_logprob_reward": 3.0509700775146484,
|
|
"eval_answer_logprob_reward/correct": 3.1185855865478516,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07136764377355576,
|
|
"eval_answer_logprob_reward/incorrect": 2.929009199142456,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.272965669631958,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.3084012567996979,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.272965669631958,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1634349822998047,
|
|
"eval_completion_length": 524.856689453125,
|
|
"eval_completion_length/correct": 461.5336608886719,
|
|
"eval_completion_length/correct/max": 1008.0,
|
|
"eval_completion_length/correct/median": 441.0,
|
|
"eval_completion_length/correct/min": 119.0,
|
|
"eval_completion_length/correct/p25": 307.0,
|
|
"eval_completion_length/correct/p75": 582.0,
|
|
"eval_completion_length/correct/var": 40730.69140625,
|
|
"eval_completion_length/incorrect": 639.0747680664062,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 580.0,
|
|
"eval_completion_length/incorrect/min": 159.0,
|
|
"eval_completion_length/incorrect/p25": 381.0,
|
|
"eval_completion_length/incorrect/p75": 997.5,
|
|
"eval_completion_length/incorrect/var": 86812.4375,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 467.0,
|
|
"eval_completion_length/min": 119.0,
|
|
"eval_completion_length/p25": 332.0,
|
|
"eval_completion_length/p75": 670.0,
|
|
"eval_completion_length/var": 64187.9609375,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 4250371.0,
|
|
"eval_len_reward": 0.037196140736341476,
|
|
"eval_len_reward/correct": 0.13615931570529938,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.14834243059158325,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.023277467116713524,
|
|
"eval_len_reward/correct/p75": 0.3070557415485382,
|
|
"eval_len_reward/correct/var": 0.05235496163368225,
|
|
"eval_len_reward/incorrect": -0.1413075029850006,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": 0.0,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2787383198738098,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03431173041462898,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.10086379945278168,
|
|
"eval_len_reward/p75": 0.21484653651714325,
|
|
"eval_len_reward/var": 0.06350763142108917,
|
|
"eval_mean_logprobs": -0.01312255859375,
|
|
"eval_mean_logprobs/var": 9.107589721679688e-05,
|
|
"eval_missing_response_penalty": -0.07999999821186066,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.22429905831813812,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.17563040554523468,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07384615391492844,
|
|
"eval_num_completions/total": 7500,
|
|
"eval_pred_value_log_probs": -0.624106228351593,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.671875,
|
|
"eval_pred_value_log_probs/min": -1.015625,
|
|
"eval_pred_value_log_probs/p25": -0.671875,
|
|
"eval_pred_value_log_probs/p75": -0.671875,
|
|
"eval_pred_value_log_probs/var": 0.026507392525672913,
|
|
"eval_reward": 0.643333375453949,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23022297024726868,
|
|
"eval_rewards/accuracy_reward": 0.643333375453949,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23022297024726868,
|
|
"eval_rewards/answer_logprob_reward": 3.0509700775146484,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.272965669631958,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1634349822998047,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.037196140736341476,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.10086379945278168,
|
|
"eval_rewards/len_reward/p75": 0.21484653651714325,
|
|
"eval_rewards/len_reward/var": 0.06350763142108917,
|
|
"eval_rewards/missing_response_penalty": -0.07999999821186066,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07384615391492844,
|
|
"eval_runtime": 166.0003,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8647846579551697,
|
|
"eval_values/max": 1.9911270141601562,
|
|
"eval_values/median": -0.82568359375,
|
|
"eval_values/min": -1.9895076751708984,
|
|
"eval_values/p25": -0.82568359375,
|
|
"eval_values/p75": -0.82568359375,
|
|
"eval_values/var": 0.1284439116716385,
|
|
"step": 25
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7916666865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.16579407453536987,
|
|
"adam_stats/lr_effective_max": 1.9133012756356038e-05,
|
|
"adam_stats/lr_effective_mean": 6.889214893002205e-11,
|
|
"adam_stats/lr_effective_min": -1.8895842003985308e-05,
|
|
"adam_stats/m_t_max": 0.00241082813590765,
|
|
"adam_stats/m_t_mean": -2.992975312032797e-11,
|
|
"adam_stats/m_t_min": -0.0015358412638306618,
|
|
"adam_stats/v_t_max": 2.3136642994359136e-05,
|
|
"adam_stats/v_t_mean": 1.5624625074664444e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7916666865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.16579407453536987,
|
|
"all_logprobs": -0.01206886488944292,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -9.0,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.02019508183002472,
|
|
"answer_logprob_reward": 3.0381431579589844,
|
|
"answer_logprob_reward/correct": 3.1016485691070557,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.05628438666462898,
|
|
"answer_logprob_reward/incorrect": 2.7968227863311768,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.319070041179657,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.12505289912223816,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 506.890625,
|
|
"completion_length/correct": 430.7105407714844,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 408.0,
|
|
"completion_length/correct/min": 122.0,
|
|
"completion_length/correct/p25": 309.25,
|
|
"completion_length/correct/p75": 521.5,
|
|
"completion_length/correct/var": 30985.37109375,
|
|
"completion_length/incorrect": 796.375,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 898.0,
|
|
"completion_length/incorrect/min": 337.0,
|
|
"completion_length/incorrect/p25": 561.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 63653.1171875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 431.0,
|
|
"completion_length/min": 122.0,
|
|
"completion_length/p25": 334.25,
|
|
"completion_length/p75": 656.25,
|
|
"completion_length/var": 59661.93359375,
|
|
"epoch": 0.6582278481012658,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2789877.0,
|
|
"grad_norm": 0.22772380709648132,
|
|
"kl": 0.013031005859375,
|
|
"learning_rate": 3.0348748417303826e-06,
|
|
"len_reward": 0.11382603645324707,
|
|
"len_reward/correct": 0.20018017292022705,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2098509967327118,
|
|
"len_reward/correct/min": -0.48064124584198,
|
|
"len_reward/correct/p25": 0.10171883553266525,
|
|
"len_reward/correct/p75": 0.37242966890335083,
|
|
"len_reward/correct/var": 0.04144986718893051,
|
|
"len_reward/incorrect": -0.21431966125965118,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1825396865606308,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.38078171014785767,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03748637065291405,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.1651490032672882,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.010968567803502083,
|
|
"len_reward/p75": 0.3135233521461487,
|
|
"len_reward/var": 0.06890864670276642,
|
|
"loss": -0.7911,
|
|
"mean_logprobs": -0.01165771484375,
|
|
"mean_logprobs/var": 7.104873657226562e-05,
|
|
"missing_response_penalty": -0.078125,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.375,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.24038463830947876,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.07239855825901031,
|
|
"non_smoothed_logprobs/min": -17.4375,
|
|
"num_completions/total": 4992,
|
|
"policy_loss": -0.7911450266838074,
|
|
"policy_loss/max": 0.0009366496815346181,
|
|
"policy_loss/median": -0.9994315505027771,
|
|
"policy_loss/min": -0.9998270869255066,
|
|
"policy_loss/p25": -0.9995665550231934,
|
|
"policy_loss/p75": -0.9991076588630676,
|
|
"policy_loss/var": 0.1657780408859253,
|
|
"pred_value_log_probs": -0.6421036720275879,
|
|
"pred_value_log_probs/max": -5.960464477539062e-07,
|
|
"pred_value_log_probs/median": -0.671875,
|
|
"pred_value_log_probs/min": -1.3203125,
|
|
"pred_value_log_probs/p25": -0.671875,
|
|
"pred_value_log_probs/p75": -0.671875,
|
|
"pred_value_log_probs/var": 0.029975350946187973,
|
|
"reward": 0.7916666865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.16579407453536987,
|
|
"rewards/accuracy_reward": 0.7916666865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.16579407453536987,
|
|
"rewards/answer_logprob_reward": 3.0381431579589844,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.12505289912223816,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.11382603645324707,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.1651490032672882,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.010968567803502083,
|
|
"rewards/len_reward/p75": 0.3135233521461487,
|
|
"rewards/len_reward/var": 0.06890864670276642,
|
|
"rewards/missing_response_penalty": -0.078125,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.07239855825901031,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 26,
|
|
"value_loss": 2.7890625,
|
|
"values": -0.8395852446556091,
|
|
"values/max": 1.30419921875,
|
|
"values/median": -0.82568359375,
|
|
"values/min": -2.000000476837158,
|
|
"values/p25": -0.82568359375,
|
|
"values/p75": -0.82568359375,
|
|
"values/var": 0.12676073610782623
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 1.9133012756356038e-05,
|
|
"adam_stats/lr_effective_mean": 6.889214893002205e-11,
|
|
"adam_stats/lr_effective_min": -1.8895842003985308e-05,
|
|
"adam_stats/m_t_max": 0.00241082813590765,
|
|
"adam_stats/m_t_mean": -2.992975312032797e-11,
|
|
"adam_stats/m_t_min": -0.0015358412638306618,
|
|
"adam_stats/v_t_max": 2.3136642994359136e-05,
|
|
"adam_stats/v_t_mean": 1.5624625074664444e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.6582278481012658,
|
|
"eval_accuracy_reward": 0.643333375453949,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23022295534610748,
|
|
"eval_advantages": 0.643333375453949,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23022295534610748,
|
|
"eval_all_logprobs": -0.013339114375412464,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -6.65625,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.019734643399715424,
|
|
"eval_answer_logprob_reward": 3.0400807857513428,
|
|
"eval_answer_logprob_reward/correct": 3.112013101577759,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06993844360113144,
|
|
"eval_answer_logprob_reward/incorrect": 2.9103331565856934,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.32598403096199036,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.16984079778194427,
|
|
"eval_completion_length": 524.2467041015625,
|
|
"eval_completion_length/correct": 455.0051574707031,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 408.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 289.0,
|
|
"eval_completion_length/correct/p75": 595.0,
|
|
"eval_completion_length/correct/var": 42489.26171875,
|
|
"eval_completion_length/incorrect": 649.14013671875,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 606.0,
|
|
"eval_completion_length/incorrect/min": 159.0,
|
|
"eval_completion_length/incorrect/p25": 428.5,
|
|
"eval_completion_length/incorrect/p75": 994.5,
|
|
"eval_completion_length/incorrect/var": 76860.953125,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 467.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 318.5,
|
|
"eval_completion_length/p75": 691.25,
|
|
"eval_completion_length/var": 63209.1640625,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 4407645.0,
|
|
"eval_len_reward": 0.04974249377846718,
|
|
"eval_len_reward/correct": 0.1533033549785614,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1989329308271408,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.006053268909454346,
|
|
"eval_len_reward/correct/p75": 0.3202614486217499,
|
|
"eval_len_reward/correct/var": 0.05350358784198761,
|
|
"eval_len_reward/incorrect": -0.1370541900396347,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.019054878503084183,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2640671133995056,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03244072198867798,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.08369508385658264,
|
|
"eval_len_reward/p75": 0.2448240965604782,
|
|
"eval_len_reward/var": 0.06526707112789154,
|
|
"eval_mean_logprobs": -0.01336669921875,
|
|
"eval_mean_logprobs/var": 9.584426879882812e-05,
|
|
"eval_missing_response_penalty": -0.0833333358168602,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.23364485800266266,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.18074414134025574,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_num_completions/total": 7800,
|
|
"eval_pred_value_log_probs": -0.6271107792854309,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.67578125,
|
|
"eval_pred_value_log_probs/min": -1.0546875,
|
|
"eval_pred_value_log_probs/p25": -0.67578125,
|
|
"eval_pred_value_log_probs/p75": -0.67578125,
|
|
"eval_pred_value_log_probs/var": 0.026666205376386642,
|
|
"eval_reward": 0.643333375453949,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23022295534610748,
|
|
"eval_rewards/accuracy_reward": 0.643333375453949,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23022295534610748,
|
|
"eval_rewards/answer_logprob_reward": 3.0400807857513428,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.16984079778194427,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.04974249377846718,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.08369508385658264,
|
|
"eval_rewards/len_reward/p75": 0.2448240965604782,
|
|
"eval_rewards/len_reward/var": 0.06526707112789154,
|
|
"eval_rewards/missing_response_penalty": -0.0833333358168602,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_runtime": 164.0283,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8602791428565979,
|
|
"eval_values/max": 1.9909400939941406,
|
|
"eval_values/median": -0.8203125,
|
|
"eval_values/min": -1.9979867935180664,
|
|
"eval_values/p25": -0.8203125,
|
|
"eval_values/p75": -0.8203125,
|
|
"eval_values/var": 0.128428652882576,
|
|
"step": 26
|
|
},
|
|
{
|
|
"accuracy_reward": 0.75,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.75,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.1884816735982895,
|
|
"adam_stats/lr_effective_max": 1.6164200133061968e-05,
|
|
"adam_stats/lr_effective_mean": 8.531608752804232e-11,
|
|
"adam_stats/lr_effective_min": -1.6297251931973733e-05,
|
|
"adam_stats/m_t_max": 0.0021214892622083426,
|
|
"adam_stats/m_t_mean": -2.8601409435280445e-11,
|
|
"adam_stats/m_t_min": -0.001608126563951373,
|
|
"adam_stats/v_t_max": 2.311374009877909e-05,
|
|
"adam_stats/v_t_mean": 1.5665066899900326e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.75,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.75,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.1884816735982895,
|
|
"all_logprobs": -0.01235611829906702,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.25,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.018512871116399765,
|
|
"answer_logprob_reward": 2.921663284301758,
|
|
"answer_logprob_reward/correct": 3.0877768993377686,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06415103375911713,
|
|
"answer_logprob_reward/incorrect": 2.4233226776123047,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/min": 1.0986123085021973,
|
|
"answer_logprob_reward/incorrect/p25": 2.007521152496338,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.6102849245071411,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.0986123085021973,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.28141871094703674,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 571.3177490234375,
|
|
"completion_length/correct": 477.0069580078125,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 476.0,
|
|
"completion_length/correct/min": 114.0,
|
|
"completion_length/correct/p25": 303.5,
|
|
"completion_length/correct/p75": 595.5,
|
|
"completion_length/correct/var": 48252.68359375,
|
|
"completion_length/incorrect": 854.25,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 1024.0,
|
|
"completion_length/incorrect/min": 198.0,
|
|
"completion_length/incorrect/p25": 720.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 59079.8515625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 522.0,
|
|
"completion_length/min": 114.0,
|
|
"completion_length/p25": 337.25,
|
|
"completion_length/p75": 827.75,
|
|
"completion_length/var": 77487.59375,
|
|
"epoch": 0.6835443037974683,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 2899570.0,
|
|
"grad_norm": 0.20783691108226776,
|
|
"kl": 0.012664794921875,
|
|
"learning_rate": 2.6306566876350072e-06,
|
|
"len_reward": 0.06911170482635498,
|
|
"len_reward/correct": 0.16831690073013306,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.20964567363262177,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.025169409811496735,
|
|
"len_reward/correct/p75": 0.3304530382156372,
|
|
"len_reward/correct/var": 0.05686808004975319,
|
|
"len_reward/incorrect": -0.22850386798381805,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.24156877398490906,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.387567937374115,
|
|
"len_reward/incorrect/p75": -0.07083163410425186,
|
|
"len_reward/incorrect/var": 0.030994582921266556,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.10745261609554291,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.12372851371765137,
|
|
"len_reward/p75": 0.2982177138328552,
|
|
"len_reward/var": 0.0798831582069397,
|
|
"loss": -0.7495,
|
|
"mean_logprobs": -0.012451171875,
|
|
"mean_logprobs/var": 6.818771362304688e-05,
|
|
"missing_response_penalty": -0.1302083432674408,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.5208333730697632,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": -1.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.25487589836120605,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.11384708434343338,
|
|
"non_smoothed_logprobs/min": -17.75,
|
|
"num_completions/total": 5184,
|
|
"policy_loss": -0.749494194984436,
|
|
"policy_loss/max": 0.0013085182290524244,
|
|
"policy_loss/median": -0.9994063377380371,
|
|
"policy_loss/min": -0.9998708367347717,
|
|
"policy_loss/p25": -0.9995794296264648,
|
|
"policy_loss/p75": -0.7491034865379333,
|
|
"policy_loss/var": 0.18845784664154053,
|
|
"pred_value_log_probs": -0.6104012727737427,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.67578125,
|
|
"pred_value_log_probs/min": -1.1328125,
|
|
"pred_value_log_probs/p25": -0.67578125,
|
|
"pred_value_log_probs/p75": -0.67578125,
|
|
"pred_value_log_probs/var": 0.04308498278260231,
|
|
"reward": 0.75,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.75,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.1884816735982895,
|
|
"rewards/accuracy_reward": 0.75,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.75,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.1884816735982895,
|
|
"rewards/answer_logprob_reward": 2.921663284301758,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.0986123085021973,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.28141871094703674,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.06911170482635498,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.10745261609554291,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.12372851371765137,
|
|
"rewards/len_reward/p75": 0.2982177138328552,
|
|
"rewards/len_reward/var": 0.0798831582069397,
|
|
"rewards/missing_response_penalty": -0.1302083432674408,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.11384708434343338,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 27,
|
|
"value_loss": 2.828125,
|
|
"values": -0.8203917741775513,
|
|
"values/max": 1.244140625,
|
|
"values/median": -0.8203125,
|
|
"values/min": -1.9633731842041016,
|
|
"values/p25": -0.8203125,
|
|
"values/p75": -0.8203125,
|
|
"values/var": 0.22798120975494385
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 1.6164200133061968e-05,
|
|
"adam_stats/lr_effective_mean": 8.531608752804232e-11,
|
|
"adam_stats/lr_effective_min": -1.6297251931973733e-05,
|
|
"adam_stats/m_t_max": 0.0021214892622083426,
|
|
"adam_stats/m_t_mean": -2.8601409435280445e-11,
|
|
"adam_stats/m_t_min": -0.001608126563951373,
|
|
"adam_stats/v_t_max": 2.311374009877909e-05,
|
|
"adam_stats/v_t_mean": 1.5665066899900326e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.6835443037974683,
|
|
"eval_accuracy_reward": 0.6366666555404663,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2320958822965622,
|
|
"eval_advantages": 0.6366666555404663,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2320958822965622,
|
|
"eval_all_logprobs": -0.012918404303491116,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -8.75,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.019511204212903976,
|
|
"eval_answer_logprob_reward": 3.075014114379883,
|
|
"eval_answer_logprob_reward/correct": 3.1354079246520996,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.051156532019376755,
|
|
"eval_answer_logprob_reward/incorrect": 2.9691860675811768,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.2690730094909668,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.13611048460006714,
|
|
"eval_completion_length": 522.8699951171875,
|
|
"eval_completion_length/correct": 452.8638916015625,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 415.0,
|
|
"eval_completion_length/correct/min": 119.0,
|
|
"eval_completion_length/correct/p25": 292.0,
|
|
"eval_completion_length/correct/p75": 570.0,
|
|
"eval_completion_length/correct/var": 42531.328125,
|
|
"eval_completion_length/incorrect": 645.541259765625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 590.0,
|
|
"eval_completion_length/incorrect/min": 170.0,
|
|
"eval_completion_length/incorrect/p25": 412.0,
|
|
"eval_completion_length/incorrect/p75": 924.0,
|
|
"eval_completion_length/incorrect/var": 77136.40625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 469.0,
|
|
"eval_completion_length/min": 119.0,
|
|
"eval_completion_length/p25": 314.75,
|
|
"eval_completion_length/p75": 670.5,
|
|
"eval_completion_length/var": 63505.03515625,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 4564506.0,
|
|
"eval_len_reward": 0.04475758224725723,
|
|
"eval_len_reward/correct": 0.15148313343524933,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1757829785346985,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.015369649976491928,
|
|
"eval_len_reward/correct/p75": 0.31522229313850403,
|
|
"eval_len_reward/correct/var": 0.052471134811639786,
|
|
"eval_len_reward/incorrect": -0.14225688576698303,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.006225680932402611,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.29405033588409424,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.031476959586143494,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.0766662061214447,
|
|
"eval_len_reward/p75": 0.23568587005138397,
|
|
"eval_len_reward/var": 0.0647384375333786,
|
|
"eval_mean_logprobs": -0.0128173828125,
|
|
"eval_mean_logprobs/var": 7.82012939453125e-05,
|
|
"eval_missing_response_penalty": -0.0833333358168602,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2293577939271927,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.17838941514492035,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_num_completions/total": 8100,
|
|
"eval_pred_value_log_probs": -0.6281352639198303,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.67578125,
|
|
"eval_pred_value_log_probs/min": -1.0625,
|
|
"eval_pred_value_log_probs/p25": -0.67578125,
|
|
"eval_pred_value_log_probs/p75": -0.67578125,
|
|
"eval_pred_value_log_probs/var": 0.026235129684209824,
|
|
"eval_reward": 0.6366666555404663,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2320958822965622,
|
|
"eval_rewards/accuracy_reward": 0.6366666555404663,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2320958822965622,
|
|
"eval_rewards/answer_logprob_reward": 3.075014114379883,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.13611048460006714,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.04475758224725723,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.0766662061214447,
|
|
"eval_rewards/len_reward/p75": 0.23568587005138397,
|
|
"eval_rewards/len_reward/var": 0.0647384375333786,
|
|
"eval_rewards/missing_response_penalty": -0.0833333358168602,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_runtime": 172.4692,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8561747670173645,
|
|
"eval_values/max": 1.9912757873535156,
|
|
"eval_values/median": -0.81640625,
|
|
"eval_values/min": -1.997899055480957,
|
|
"eval_values/p25": -0.81640625,
|
|
"eval_values/p75": -0.81640625,
|
|
"eval_values/var": 0.1274670958518982,
|
|
"step": 27
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7864583730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.16882090270519257,
|
|
"adam_stats/lr_effective_max": 1.4186392036208417e-05,
|
|
"adam_stats/lr_effective_mean": 7.851148897675131e-11,
|
|
"adam_stats/lr_effective_min": -1.3711563042306807e-05,
|
|
"adam_stats/m_t_max": 0.002043617656454444,
|
|
"adam_stats/m_t_mean": -2.1564961621978185e-11,
|
|
"adam_stats/m_t_min": -0.0016183295520022511,
|
|
"adam_stats/v_t_max": 2.3092428818927146e-05,
|
|
"adam_stats/v_t_mean": 1.5696143386770278e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7864583730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.16882090270519257,
|
|
"all_logprobs": -0.011460821144282818,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -8.375,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.015566036105155945,
|
|
"answer_logprob_reward": 3.047133207321167,
|
|
"answer_logprob_reward/correct": 3.0919618606567383,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.061520010232925415,
|
|
"answer_logprob_reward/incorrect": 2.882032871246338,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.18731394410133362,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 2.079441547393799,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.09498215466737747,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 524.15625,
|
|
"completion_length/correct": 471.0860900878906,
|
|
"completion_length/correct/max": 998.0,
|
|
"completion_length/correct/median": 439.0,
|
|
"completion_length/correct/min": 71.0,
|
|
"completion_length/correct/p25": 333.0,
|
|
"completion_length/correct/p75": 582.0,
|
|
"completion_length/correct/var": 40626.06640625,
|
|
"completion_length/incorrect": 719.6097412109375,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 698.0,
|
|
"completion_length/incorrect/min": 321.0,
|
|
"completion_length/incorrect/p25": 435.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 74266.4921875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 471.0,
|
|
"completion_length/min": 71.0,
|
|
"completion_length/p25": 359.75,
|
|
"completion_length/p75": 684.0,
|
|
"completion_length/var": 57885.53515625,
|
|
"epoch": 0.7088607594936709,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3000208.0,
|
|
"grad_norm": 0.22927089035511017,
|
|
"kl": 0.0133056640625,
|
|
"learning_rate": 2.245515092739488e-06,
|
|
"len_reward": 0.05197674781084061,
|
|
"len_reward/correct": 0.12103062123060226,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.16318106651306152,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.08399999886751175,
|
|
"len_reward/correct/p75": 0.3286933898925781,
|
|
"len_reward/correct/var": 0.06610661745071411,
|
|
"len_reward/incorrect": -0.20234361290931702,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.18196457624435425,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.37749120593070984,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03964154049754143,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.04279015213251114,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.16771990060806274,
|
|
"len_reward/p75": 0.2686336934566498,
|
|
"len_reward/var": 0.07787184417247772,
|
|
"loss": -0.7859,
|
|
"mean_logprobs": -0.01220703125,
|
|
"mean_logprobs/var": 8.20159912109375e-05,
|
|
"missing_response_penalty": -0.078125,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.3658536374568939,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.23780488967895508,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.07239855825901031,
|
|
"non_smoothed_logprobs/min": -17.375,
|
|
"num_completions/total": 5376,
|
|
"policy_loss": -0.785926342010498,
|
|
"policy_loss/max": 0.0010675566736608744,
|
|
"policy_loss/median": -0.9994320273399353,
|
|
"policy_loss/min": -0.9997646808624268,
|
|
"policy_loss/p25": -0.99957275390625,
|
|
"policy_loss/p75": -0.9990478754043579,
|
|
"policy_loss/var": 0.1688149869441986,
|
|
"pred_value_log_probs": -0.602538526058197,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.67578125,
|
|
"pred_value_log_probs/min": -1.03125,
|
|
"pred_value_log_probs/p25": -0.67578125,
|
|
"pred_value_log_probs/p75": -0.67578125,
|
|
"pred_value_log_probs/var": 0.04960930347442627,
|
|
"reward": 0.7864583730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.16882090270519257,
|
|
"rewards/accuracy_reward": 0.7864583730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.16882090270519257,
|
|
"rewards/answer_logprob_reward": 3.047133207321167,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 2.079441547393799,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.09498215466737747,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.05197674781084061,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.04279015213251114,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.16771990060806274,
|
|
"rewards/len_reward/p75": 0.2686336934566498,
|
|
"rewards/len_reward/var": 0.07787184417247772,
|
|
"rewards/missing_response_penalty": -0.078125,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.07239855825901031,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 28,
|
|
"value_loss": 2.8828125,
|
|
"values": -0.8180979490280151,
|
|
"values/max": 2.0,
|
|
"values/median": -0.81640625,
|
|
"values/min": -1.9906539916992188,
|
|
"values/p25": -0.81640625,
|
|
"values/p75": -0.81640625,
|
|
"values/var": 0.294994056224823
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 1.4186392036208417e-05,
|
|
"adam_stats/lr_effective_mean": 7.851148897675131e-11,
|
|
"adam_stats/lr_effective_min": -1.3711563042306807e-05,
|
|
"adam_stats/m_t_max": 0.002043617656454444,
|
|
"adam_stats/m_t_mean": -2.1564961621978185e-11,
|
|
"adam_stats/m_t_min": -0.0016183295520022511,
|
|
"adam_stats/v_t_max": 2.3092428818927146e-05,
|
|
"adam_stats/v_t_mean": 1.5696143386770278e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.7088607594936709,
|
|
"eval_accuracy_reward": 0.6600000262260437,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22515049576759338,
|
|
"eval_advantages": 0.6600000262260437,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22515049576759338,
|
|
"eval_all_logprobs": -0.013635395094752312,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.8125,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.022605856880545616,
|
|
"eval_answer_logprob_reward": 3.0406923294067383,
|
|
"eval_answer_logprob_reward/correct": 3.1021616458892822,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07666410505771637,
|
|
"eval_answer_logprob_reward/incorrect": 2.9213695526123047,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.23875315487384796,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.13851943612098694,
|
|
"eval_completion_length": 522.07666015625,
|
|
"eval_completion_length/correct": 465.57574462890625,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 419.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 292.0,
|
|
"eval_completion_length/correct/p75": 600.0,
|
|
"eval_completion_length/correct/var": 45138.41015625,
|
|
"eval_completion_length/incorrect": 631.7549438476562,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 577.0,
|
|
"eval_completion_length/incorrect/min": 164.0,
|
|
"eval_completion_length/incorrect/p25": 380.25,
|
|
"eval_completion_length/incorrect/p75": 956.25,
|
|
"eval_completion_length/incorrect/var": 84311.40625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 476.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 311.75,
|
|
"eval_completion_length/p75": 667.75,
|
|
"eval_completion_length/var": 64437.4375,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 4721129.0,
|
|
"eval_len_reward": 0.07318604737520218,
|
|
"eval_len_reward/correct": 0.17031632363796234,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.19773392379283905,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.037893105298280716,
|
|
"eval_len_reward/correct/p75": 0.321541965007782,
|
|
"eval_len_reward/correct/var": 0.044649749994277954,
|
|
"eval_len_reward/incorrect": -0.11536100506782532,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": 0.0,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.20145376026630402,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.02995114028453827,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.026206322014331818,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.021526621654629707,
|
|
"eval_len_reward/p75": 0.25351065397262573,
|
|
"eval_len_reward/var": 0.05791022256016731,
|
|
"eval_mean_logprobs": -0.0135498046875,
|
|
"eval_mean_logprobs/var": 0.00012159347534179688,
|
|
"eval_missing_response_penalty": -0.0833333358168602,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2450980544090271,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.1868569254875183,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_num_completions/total": 8400,
|
|
"eval_pred_value_log_probs": -0.6240668892860413,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.671875,
|
|
"eval_pred_value_log_probs/min": -1.0546875,
|
|
"eval_pred_value_log_probs/p25": -0.671875,
|
|
"eval_pred_value_log_probs/p75": -0.671875,
|
|
"eval_pred_value_log_probs/var": 0.026030590757727623,
|
|
"eval_reward": 0.6600000262260437,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22515049576759338,
|
|
"eval_rewards/accuracy_reward": 0.6600000262260437,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22515049576759338,
|
|
"eval_rewards/answer_logprob_reward": 3.0406923294067383,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.13851943612098694,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.07318604737520218,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.026206322014331818,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.021526621654629707,
|
|
"eval_rewards/len_reward/p75": 0.25351065397262573,
|
|
"eval_rewards/len_reward/var": 0.05791022256016731,
|
|
"eval_rewards/missing_response_penalty": -0.0833333358168602,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07664437592029572,
|
|
"eval_runtime": 169.8675,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8697391748428345,
|
|
"eval_values/max": 1.9914169311523438,
|
|
"eval_values/median": -0.83154296875,
|
|
"eval_values/min": -1.9982261657714844,
|
|
"eval_values/p25": -0.83154296875,
|
|
"eval_values/p75": -0.83154296875,
|
|
"eval_values/var": 0.1275765597820282,
|
|
"step": 28
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7291666865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.1985165923833847,
|
|
"adam_stats/lr_effective_max": 1.1584174899326172e-05,
|
|
"adam_stats/lr_effective_mean": 5.095042443503672e-11,
|
|
"adam_stats/lr_effective_min": -1.149139461631421e-05,
|
|
"adam_stats/m_t_max": 0.0020437235943973064,
|
|
"adam_stats/m_t_mean": -5.718687676181666e-12,
|
|
"adam_stats/m_t_min": -0.0016945337411016226,
|
|
"adam_stats/v_t_max": 2.307351678609848e-05,
|
|
"adam_stats/v_t_mean": 1.5742044169944625e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7291666865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.1985165923833847,
|
|
"all_logprobs": -0.012813528068363667,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -12.625,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.02218828722834587,
|
|
"answer_logprob_reward": 2.97220516204834,
|
|
"answer_logprob_reward/correct": 3.049698829650879,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 1.7917594909667969,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.09595181047916412,
|
|
"answer_logprob_reward/incorrect": 2.763568639755249,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.3107951283454895,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.1690686196088791,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 541.4739990234375,
|
|
"completion_length/correct": 475.8642883300781,
|
|
"completion_length/correct/max": 1004.0,
|
|
"completion_length/correct/median": 416.0,
|
|
"completion_length/correct/min": 102.0,
|
|
"completion_length/correct/p25": 314.75,
|
|
"completion_length/correct/p75": 605.75,
|
|
"completion_length/correct/var": 52804.36328125,
|
|
"completion_length/incorrect": 718.1154174804688,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 700.0,
|
|
"completion_length/incorrect/min": 203.0,
|
|
"completion_length/incorrect/p25": 459.75,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 80780.6171875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 459.0,
|
|
"completion_length/min": 102.0,
|
|
"completion_length/p25": 336.75,
|
|
"completion_length/p75": 749.25,
|
|
"completion_length/var": 71648.0625,
|
|
"epoch": 0.7341772151898734,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3104171.0,
|
|
"grad_norm": 0.22333048284053802,
|
|
"kl": 0.01397705078125,
|
|
"learning_rate": 1.8825509907063328e-06,
|
|
"len_reward": 0.05953149124979973,
|
|
"len_reward/correct": 0.14699342846870422,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.19756481051445007,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.02327059395611286,
|
|
"len_reward/correct/p75": 0.323266863822937,
|
|
"len_reward/correct/var": 0.05901362746953964,
|
|
"len_reward/incorrect": -0.17594295740127563,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.11352656781673431,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3106473684310913,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03516513109207153,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.017633674666285515,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.11306626349687576,
|
|
"len_reward/p75": 0.2794128358364105,
|
|
"len_reward/var": 0.07303960621356964,
|
|
"loss": -0.7286,
|
|
"mean_logprobs": -0.0126953125,
|
|
"mean_logprobs/var": 8.249282836914062e-05,
|
|
"missing_response_penalty": -0.1041666716337204,
|
|
"missing_response_penalty/correct": -0.0071428571827709675,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": -1.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0071428571827709675,
|
|
"missing_response_penalty/incorrect": -0.36538463830947876,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.23642534017562866,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.09380455315113068,
|
|
"non_smoothed_logprobs/min": -17.625,
|
|
"num_completions/total": 5568,
|
|
"policy_loss": -0.728607177734375,
|
|
"policy_loss/max": 0.0011987847974523902,
|
|
"policy_loss/median": -0.9993556141853333,
|
|
"policy_loss/min": -0.9998286366462708,
|
|
"policy_loss/p25": -0.9995555877685547,
|
|
"policy_loss/p75": 0.00018846243619918823,
|
|
"policy_loss/var": 0.19850312173366547,
|
|
"pred_value_log_probs": -0.6303989291191101,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.671875,
|
|
"pred_value_log_probs/min": -1.21875,
|
|
"pred_value_log_probs/p25": -0.671875,
|
|
"pred_value_log_probs/p75": -0.671875,
|
|
"pred_value_log_probs/var": 0.037602685391902924,
|
|
"reward": 0.7291666865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.1985165923833847,
|
|
"rewards/accuracy_reward": 0.7291666865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.1985165923833847,
|
|
"rewards/answer_logprob_reward": 2.97220516204834,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.1690686196088791,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.05953149124979973,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.017633674666285515,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.11306626349687576,
|
|
"rewards/len_reward/p75": 0.2794128358364105,
|
|
"rewards/len_reward/var": 0.07303960621356964,
|
|
"rewards/missing_response_penalty": -0.1041666716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.09380455315113068,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 29,
|
|
"value_loss": 3.015625,
|
|
"values": -0.7812522649765015,
|
|
"values/max": 1.9557266235351562,
|
|
"values/median": -0.83154296875,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.83154296875,
|
|
"values/p75": -0.83154296875,
|
|
"values/var": 0.23247919976711273
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 1.1584174899326172e-05,
|
|
"adam_stats/lr_effective_mean": 5.095042443503672e-11,
|
|
"adam_stats/lr_effective_min": -1.149139461631421e-05,
|
|
"adam_stats/m_t_max": 0.0020437235943973064,
|
|
"adam_stats/m_t_mean": -5.718687676181666e-12,
|
|
"adam_stats/m_t_min": -0.0016945337411016226,
|
|
"adam_stats/v_t_max": 2.307351678609848e-05,
|
|
"adam_stats/v_t_mean": 1.5742044169944625e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.7341772151898734,
|
|
"eval_accuracy_reward": 0.643333375453949,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23022295534610748,
|
|
"eval_advantages": 0.643333375453949,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23022295534610748,
|
|
"eval_all_logprobs": -0.013509598560631275,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -6.75,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.020643368363380432,
|
|
"eval_answer_logprob_reward": 3.034558057785034,
|
|
"eval_answer_logprob_reward/correct": 3.104830265045166,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07334812730550766,
|
|
"eval_answer_logprob_reward/incorrect": 2.907804489135742,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.2928704023361206,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.15986381471157074,
|
|
"eval_completion_length": 523.1600341796875,
|
|
"eval_completion_length/correct": 457.2124328613281,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 419.0,
|
|
"eval_completion_length/correct/min": 119.0,
|
|
"eval_completion_length/correct/p25": 288.0,
|
|
"eval_completion_length/correct/p75": 598.0,
|
|
"eval_completion_length/correct/var": 44095.76171875,
|
|
"eval_completion_length/incorrect": 642.1121215820312,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 616.0,
|
|
"eval_completion_length/incorrect/min": 159.0,
|
|
"eval_completion_length/incorrect/p25": 397.5,
|
|
"eval_completion_length/incorrect/p75": 970.5,
|
|
"eval_completion_length/incorrect/var": 81015.578125,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 469.0,
|
|
"eval_completion_length/min": 119.0,
|
|
"eval_completion_length/p25": 313.5,
|
|
"eval_completion_length/p75": 689.75,
|
|
"eval_completion_length/var": 64907.7578125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 4878077.0,
|
|
"eval_len_reward": 0.047344304621219635,
|
|
"eval_len_reward/correct": 0.15471094846725464,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.1744791716337204,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.0014566642930731177,
|
|
"eval_len_reward/correct/p75": 0.3337520956993103,
|
|
"eval_len_reward/correct/var": 0.05422097072005272,
|
|
"eval_len_reward/incorrect": -0.14631706476211548,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.059578947722911835,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2804742455482483,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03172319009900093,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.08517948538064957,
|
|
"eval_len_reward/p75": 0.24898335337638855,
|
|
"eval_len_reward/var": 0.06692613661289215,
|
|
"eval_mean_logprobs": -0.0133056640625,
|
|
"eval_mean_logprobs/var": 0.0001239776611328125,
|
|
"eval_missing_response_penalty": -0.07666666805744171,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2149532586336136,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.17034032940864563,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07102563977241516,
|
|
"eval_num_completions/total": 8700,
|
|
"eval_pred_value_log_probs": -0.6311616897583008,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6796875,
|
|
"eval_pred_value_log_probs/min": -1.09375,
|
|
"eval_pred_value_log_probs/p25": -0.6796875,
|
|
"eval_pred_value_log_probs/p75": -0.6796875,
|
|
"eval_pred_value_log_probs/var": 0.026840047910809517,
|
|
"eval_reward": 0.643333375453949,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23022295534610748,
|
|
"eval_rewards/accuracy_reward": 0.643333375453949,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23022295534610748,
|
|
"eval_rewards/answer_logprob_reward": 3.034558057785034,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.15986381471157074,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.047344304621219635,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.08517948538064957,
|
|
"eval_rewards/len_reward/p75": 0.24898335337638855,
|
|
"eval_rewards/len_reward/var": 0.06692613661289215,
|
|
"eval_rewards/missing_response_penalty": -0.07666666805744171,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07102563977241516,
|
|
"eval_runtime": 183.1127,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8550611734390259,
|
|
"eval_values/max": 1.9912834167480469,
|
|
"eval_values/median": -0.81640625,
|
|
"eval_values/min": -1.9976587295532227,
|
|
"eval_values/p25": -0.81640625,
|
|
"eval_values/p75": -0.81640625,
|
|
"eval_values/var": 0.1290351301431656,
|
|
"step": 29
|
|
},
|
|
{
|
|
"accuracy_reward": 0.75,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.75,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.1884816735982895,
|
|
"adam_stats/lr_effective_max": 9.796427548280917e-06,
|
|
"adam_stats/lr_effective_mean": 3.373353679725355e-11,
|
|
"adam_stats/lr_effective_min": -9.434922503714915e-06,
|
|
"adam_stats/m_t_max": 0.0022849079687148333,
|
|
"adam_stats/m_t_mean": 3.377608522731057e-12,
|
|
"adam_stats/m_t_min": -0.0018363596173003316,
|
|
"adam_stats/v_t_max": 2.30702953558648e-05,
|
|
"adam_stats/v_t_mean": 1.576915681367197e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.75,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.75,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.1884816735982895,
|
|
"all_logprobs": -0.012992546893656254,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.75,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.020218783989548683,
|
|
"answer_logprob_reward": 3.098630905151367,
|
|
"answer_logprob_reward/correct": 3.1202919483184814,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.03695792704820633,
|
|
"answer_logprob_reward/incorrect": 3.0336482524871826,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p25": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.08092737942934036,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 2.4849066734313965,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.048999086022377014,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 497.91668701171875,
|
|
"completion_length/correct": 439.1944580078125,
|
|
"completion_length/correct/max": 993.0,
|
|
"completion_length/correct/median": 390.0,
|
|
"completion_length/correct/min": 161.0,
|
|
"completion_length/correct/p25": 296.75,
|
|
"completion_length/correct/p75": 547.0,
|
|
"completion_length/correct/var": 37253.359375,
|
|
"completion_length/incorrect": 674.0833740234375,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 682.0,
|
|
"completion_length/incorrect/min": 202.0,
|
|
"completion_length/incorrect/p25": 395.0,
|
|
"completion_length/incorrect/p75": 980.5,
|
|
"completion_length/incorrect/var": 76959.8671875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 436.0,
|
|
"completion_length/min": 161.0,
|
|
"completion_length/p25": 309.75,
|
|
"completion_length/p75": 659.0,
|
|
"completion_length/var": 57228.09375,
|
|
"epoch": 0.759493670886076,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3199771.0,
|
|
"grad_norm": 0.21546195447444916,
|
|
"kl": 0.0135498046875,
|
|
"learning_rate": 1.544686755065677e-06,
|
|
"len_reward": 0.11244390904903412,
|
|
"len_reward/correct": 0.20514892041683197,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.23633748292922974,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.10495449602603912,
|
|
"len_reward/correct/p75": 0.3914213478565216,
|
|
"len_reward/correct/var": 0.05474286526441574,
|
|
"len_reward/incorrect": -0.1656711846590042,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.08173579722642899,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.32875633239746094,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.037785883992910385,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.16490891575813293,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.031448401510715485,
|
|
"len_reward/p75": 0.343137264251709,
|
|
"len_reward/var": 0.07620125263929367,
|
|
"loss": -0.7495,
|
|
"mean_logprobs": -0.0128173828125,
|
|
"mean_logprobs/var": 0.00020694732666015625,
|
|
"missing_response_penalty": -0.0625,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.25,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -0.25,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.19148936867713928,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.05890052393078804,
|
|
"non_smoothed_logprobs/min": -17.375,
|
|
"num_completions/total": 5760,
|
|
"policy_loss": -0.7494577765464783,
|
|
"policy_loss/max": 0.0012080126907676458,
|
|
"policy_loss/median": -0.9994379281997681,
|
|
"policy_loss/min": -0.999779462814331,
|
|
"policy_loss/p25": -0.9995537996292114,
|
|
"policy_loss/p75": -0.7489566802978516,
|
|
"policy_loss/var": 0.18848688900470734,
|
|
"pred_value_log_probs": -0.6227776408195496,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6796875,
|
|
"pred_value_log_probs/min": -1.2265625,
|
|
"pred_value_log_probs/p25": -0.6796875,
|
|
"pred_value_log_probs/p75": -0.6796875,
|
|
"pred_value_log_probs/var": 0.03195691853761673,
|
|
"reward": 0.75,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.75,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.1884816735982895,
|
|
"rewards/accuracy_reward": 0.75,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.75,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.1884816735982895,
|
|
"rewards/answer_logprob_reward": 3.098630905151367,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 2.4849066734313965,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.048999086022377014,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.11244390904903412,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.16490891575813293,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.031448401510715485,
|
|
"rewards/len_reward/p75": 0.343137264251709,
|
|
"rewards/len_reward/var": 0.07620125263929367,
|
|
"rewards/missing_response_penalty": -0.0625,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.05890052393078804,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 30,
|
|
"value_loss": 2.828125,
|
|
"values": -0.8329395651817322,
|
|
"values/max": 1.816192626953125,
|
|
"values/median": -0.81640625,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.81640625,
|
|
"values/p75": -0.81640625,
|
|
"values/var": 0.17554526031017303
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 9.796427548280917e-06,
|
|
"adam_stats/lr_effective_mean": 3.373353679725355e-11,
|
|
"adam_stats/lr_effective_min": -9.434922503714915e-06,
|
|
"adam_stats/m_t_max": 0.0022849079687148333,
|
|
"adam_stats/m_t_mean": 3.377608522731057e-12,
|
|
"adam_stats/m_t_min": -0.0018363596173003316,
|
|
"adam_stats/v_t_max": 2.30702953558648e-05,
|
|
"adam_stats/v_t_mean": 1.576915681367197e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.759493670886076,
|
|
"eval_accuracy_reward": 0.6166666746139526,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2371794730424881,
|
|
"eval_advantages": 0.6166666746139526,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2371794730424881,
|
|
"eval_all_logprobs": -0.015801355242729187,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.25,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.025429002940654755,
|
|
"eval_answer_logprob_reward": 3.0262176990509033,
|
|
"eval_answer_logprob_reward/correct": 3.117713689804077,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.060211893171072006,
|
|
"eval_answer_logprob_reward/incorrect": 2.879028081893921,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.27812936902046204,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.15660837292671204,
|
|
"eval_completion_length": 535.6366577148438,
|
|
"eval_completion_length/correct": 458.4432678222656,
|
|
"eval_completion_length/correct/max": 1005.0,
|
|
"eval_completion_length/correct/median": 409.0,
|
|
"eval_completion_length/correct/min": 120.0,
|
|
"eval_completion_length/correct/p25": 296.0,
|
|
"eval_completion_length/correct/p75": 599.0,
|
|
"eval_completion_length/correct/var": 43624.09375,
|
|
"eval_completion_length/incorrect": 659.8173828125,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 609.0,
|
|
"eval_completion_length/incorrect/min": 161.0,
|
|
"eval_completion_length/incorrect/p25": 406.5,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 85657.09375,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 465.0,
|
|
"eval_completion_length/min": 120.0,
|
|
"eval_completion_length/p25": 318.0,
|
|
"eval_completion_length/p75": 715.5,
|
|
"eval_completion_length/var": 69122.15625,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 5038768.0,
|
|
"eval_len_reward": 0.019363053143024445,
|
|
"eval_len_reward/correct": 0.13039062917232513,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.15446966886520386,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.04123222827911377,
|
|
"eval_len_reward/correct/p75": 0.3129418194293976,
|
|
"eval_len_reward/correct/var": 0.05886157602071762,
|
|
"eval_len_reward/incorrect": -0.1592465192079544,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.035070907324552536,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.30080854892730713,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03488873317837715,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.1431734263896942,
|
|
"eval_len_reward/p75": 0.23216068744659424,
|
|
"eval_len_reward/var": 0.06942147761583328,
|
|
"eval_mean_logprobs": -0.01495361328125,
|
|
"eval_mean_logprobs/var": 0.000659942626953125,
|
|
"eval_missing_response_penalty": -0.10000000149011612,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.260869562625885,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.19450801610946655,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.09030099958181381,
|
|
"eval_num_completions/total": 9000,
|
|
"eval_pred_value_log_probs": -0.6379637122154236,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6875,
|
|
"eval_pred_value_log_probs/min": -1.0,
|
|
"eval_pred_value_log_probs/p25": -0.6875,
|
|
"eval_pred_value_log_probs/p75": -0.6875,
|
|
"eval_pred_value_log_probs/var": 0.026982415467500687,
|
|
"eval_reward": 0.6166666746139526,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2371794730424881,
|
|
"eval_rewards/accuracy_reward": 0.6166666746139526,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2371794730424881,
|
|
"eval_rewards/answer_logprob_reward": 3.0262176990509033,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.15660837292671204,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.019363053143024445,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.1431734263896942,
|
|
"eval_rewards/len_reward/p75": 0.23216068744659424,
|
|
"eval_rewards/len_reward/var": 0.06942147761583328,
|
|
"eval_rewards/missing_response_penalty": -0.10000000149011612,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.09030099958181381,
|
|
"eval_runtime": 165.6023,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8475949168205261,
|
|
"eval_values/max": 1.9912223815917969,
|
|
"eval_values/median": -0.8076171875,
|
|
"eval_values/min": -1.9976463317871094,
|
|
"eval_values/p25": -0.8076171875,
|
|
"eval_values/p75": -0.8076171875,
|
|
"eval_values/var": 0.12839378416538239,
|
|
"step": 30
|
|
},
|
|
{
|
|
"accuracy_reward": 0.734375,
|
|
"accuracy_reward/correct": 0.9999999403953552,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.19608965516090393,
|
|
"adam_stats/lr_effective_max": 7.4884374043904245e-06,
|
|
"adam_stats/lr_effective_mean": 2.6002009573256046e-11,
|
|
"adam_stats/lr_effective_min": -7.4642371146182995e-06,
|
|
"adam_stats/m_t_max": 0.002186116762459278,
|
|
"adam_stats/m_t_mean": 1.3758061530311227e-11,
|
|
"adam_stats/m_t_min": -0.0015871108043938875,
|
|
"adam_stats/v_t_max": 2.304890767845791e-05,
|
|
"adam_stats/v_t_mean": 1.5836876081365414e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.734375,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.19608965516090393,
|
|
"all_logprobs": -0.017007239162921906,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -10.8125,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.029502060264348984,
|
|
"answer_logprob_reward": 2.951953649520874,
|
|
"answer_logprob_reward/correct": 3.1006038188934326,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.05757172405719757,
|
|
"answer_logprob_reward/incorrect": 2.5409796237945557,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.3862943649291992,
|
|
"answer_logprob_reward/incorrect/p25": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.5237216353416443,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.3862943649291992,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.2407103180885315,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 557.90625,
|
|
"completion_length/correct": 470.1631164550781,
|
|
"completion_length/correct/max": 999.0,
|
|
"completion_length/correct/median": 438.0,
|
|
"completion_length/correct/min": 109.0,
|
|
"completion_length/correct/p25": 288.0,
|
|
"completion_length/correct/p75": 632.0,
|
|
"completion_length/correct/var": 48596.6484375,
|
|
"completion_length/incorrect": 800.490234375,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 921.0,
|
|
"completion_length/incorrect/min": 227.0,
|
|
"completion_length/incorrect/p25": 573.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 67807.0546875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 493.0,
|
|
"completion_length/min": 109.0,
|
|
"completion_length/p25": 321.5,
|
|
"completion_length/p75": 798.5,
|
|
"completion_length/var": 74767.6328125,
|
|
"epoch": 0.7848101265822784,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3306889.0,
|
|
"grad_norm": 0.24691376090049744,
|
|
"kl": 0.013580322265625,
|
|
"learning_rate": 1.234642669981946e-06,
|
|
"len_reward": 0.08584549278020859,
|
|
"len_reward/correct": 0.18430396914482117,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.22412508726119995,
|
|
"len_reward/correct/min": -0.4765070974826813,
|
|
"len_reward/correct/p25": 0.04035968333482742,
|
|
"len_reward/correct/p75": 0.359270304441452,
|
|
"len_reward/correct/var": 0.050723738968372345,
|
|
"len_reward/incorrect": -0.18636326491832733,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.13509193062782288,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.33006739616394043,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.035194478929042816,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.07579787075519562,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.08782581239938736,
|
|
"len_reward/p75": 0.3012104034423828,
|
|
"len_reward/var": 0.07333450019359589,
|
|
"loss": -0.7338,
|
|
"mean_logprobs": -0.0152587890625,
|
|
"mean_logprobs/var": 0.000904083251953125,
|
|
"missing_response_penalty": -0.125,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4705882668495178,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2541176676750183,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.10994765162467957,
|
|
"non_smoothed_logprobs/min": -17.5,
|
|
"num_completions/total": 5952,
|
|
"policy_loss": -0.7338322401046753,
|
|
"policy_loss/max": 0.0009722772520035505,
|
|
"policy_loss/median": -0.9993570446968079,
|
|
"policy_loss/min": -0.9997707605361938,
|
|
"policy_loss/p25": -0.9995685815811157,
|
|
"policy_loss/p75": 0.0002435501664876938,
|
|
"policy_loss/var": 0.19606992602348328,
|
|
"pred_value_log_probs": -0.6232706904411316,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6875,
|
|
"pred_value_log_probs/min": -1.3046875,
|
|
"pred_value_log_probs/p25": -0.6875,
|
|
"pred_value_log_probs/p75": -0.6875,
|
|
"pred_value_log_probs/var": 0.04709911346435547,
|
|
"reward": 0.734375,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.19608965516090393,
|
|
"rewards/accuracy_reward": 0.734375,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.19608965516090393,
|
|
"rewards/answer_logprob_reward": 2.951953649520874,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.3862943649291992,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.2407103180885315,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.08584549278020859,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.07579787075519562,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.08782581239938736,
|
|
"rewards/len_reward/p75": 0.3012104034423828,
|
|
"rewards/len_reward/var": 0.07333450019359589,
|
|
"rewards/missing_response_penalty": -0.125,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.10994765162467957,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 31,
|
|
"value_loss": 2.8203125,
|
|
"values": -0.8314536809921265,
|
|
"values/max": 1.558349609375,
|
|
"values/median": -0.8076171875,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.8076171875,
|
|
"values/p75": -0.8076171875,
|
|
"values/var": 0.21946369111537933
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 7.4884374043904245e-06,
|
|
"adam_stats/lr_effective_mean": 2.6002009573256046e-11,
|
|
"adam_stats/lr_effective_min": -7.4642371146182995e-06,
|
|
"adam_stats/m_t_max": 0.002186116762459278,
|
|
"adam_stats/m_t_mean": 1.3758061530311227e-11,
|
|
"adam_stats/m_t_min": -0.0015871108043938875,
|
|
"adam_stats/v_t_max": 2.304890767845791e-05,
|
|
"adam_stats/v_t_mean": 1.5836876081365414e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.7848101265822784,
|
|
"eval_accuracy_reward": 0.6466667056083679,
|
|
"eval_accuracy_reward/correct": 0.9999999403953552,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.22925306856632233,
|
|
"eval_advantages": 0.6466667056083679,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.22925306856632233,
|
|
"eval_all_logprobs": -0.013432860374450684,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -9.0,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02059178426861763,
|
|
"eval_answer_logprob_reward": 3.0655412673950195,
|
|
"eval_answer_logprob_reward/correct": 3.1116878986358643,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06313309818506241,
|
|
"eval_answer_logprob_reward/incorrect": 2.981083869934082,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.22886857390403748,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.12503387033939362,
|
|
"eval_completion_length": 517.8833618164062,
|
|
"eval_completion_length/correct": 459.0257568359375,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 432.0,
|
|
"eval_completion_length/correct/min": 123.0,
|
|
"eval_completion_length/correct/p25": 295.75,
|
|
"eval_completion_length/correct/p75": 579.75,
|
|
"eval_completion_length/correct/var": 42457.7578125,
|
|
"eval_completion_length/incorrect": 625.603759765625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 579.0,
|
|
"eval_completion_length/incorrect/min": 161.0,
|
|
"eval_completion_length/incorrect/p25": 396.25,
|
|
"eval_completion_length/incorrect/p75": 965.75,
|
|
"eval_completion_length/incorrect/var": 77964.5078125,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 474.0,
|
|
"eval_completion_length/min": 123.0,
|
|
"eval_completion_length/p25": 313.0,
|
|
"eval_completion_length/p75": 649.25,
|
|
"eval_completion_length/var": 61146.046875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 5194133.0,
|
|
"eval_len_reward": 0.059598326683044434,
|
|
"eval_len_reward/correct": 0.15640662610530853,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.17131148278713226,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.025762856006622314,
|
|
"eval_len_reward/correct/p75": 0.3303588032722473,
|
|
"eval_len_reward/correct/var": 0.04956372454762459,
|
|
"eval_len_reward/incorrect": -0.1175791323184967,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.016791781410574913,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.19059230387210846,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.027757765725255013,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.004508196841925383,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.06665706634521484,
|
|
"eval_len_reward/p75": 0.245398610830307,
|
|
"eval_len_reward/var": 0.05894996598362923,
|
|
"eval_mean_logprobs": -0.0135498046875,
|
|
"eval_mean_logprobs/var": 0.0001068115234375,
|
|
"eval_missing_response_penalty": -0.05666666850447655,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.1603773534297943,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.13593891263008118,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.053634338080883026,
|
|
"eval_num_completions/total": 9300,
|
|
"eval_pred_value_log_probs": -0.6279247403144836,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.67578125,
|
|
"eval_pred_value_log_probs/min": -1.078125,
|
|
"eval_pred_value_log_probs/p25": -0.67578125,
|
|
"eval_pred_value_log_probs/p75": -0.67578125,
|
|
"eval_pred_value_log_probs/var": 0.02614113688468933,
|
|
"eval_reward": 0.6466667056083679,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.22925306856632233,
|
|
"eval_rewards/accuracy_reward": 0.6466667056083679,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.22925306856632233,
|
|
"eval_rewards/answer_logprob_reward": 3.0655412673950195,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.12503387033939362,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.059598326683044434,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.004508196841925383,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.06665706634521484,
|
|
"eval_rewards/len_reward/p75": 0.245398610830307,
|
|
"eval_rewards/len_reward/var": 0.05894996598362923,
|
|
"eval_rewards/missing_response_penalty": -0.05666666850447655,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.053634338080883026,
|
|
"eval_runtime": 165.3245,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.859175443649292,
|
|
"eval_values/max": 1.9910888671875,
|
|
"eval_values/median": -0.82080078125,
|
|
"eval_values/min": -1.9976825714111328,
|
|
"eval_values/p25": -0.82080078125,
|
|
"eval_values/p75": -0.82080078125,
|
|
"eval_values/var": 0.12633655965328217,
|
|
"step": 31
|
|
},
|
|
{
|
|
"accuracy_reward": 0.8072916865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.1563863456249237,
|
|
"adam_stats/lr_effective_max": 5.642576979880687e-06,
|
|
"adam_stats/lr_effective_mean": 1.6730929142116935e-11,
|
|
"adam_stats/lr_effective_min": -5.89284081797814e-06,
|
|
"adam_stats/m_t_max": 0.0029257573187351227,
|
|
"adam_stats/m_t_mean": 3.1172169880078204e-12,
|
|
"adam_stats/m_t_min": -0.0021233742590993643,
|
|
"adam_stats/v_t_max": 2.3117683667805977e-05,
|
|
"adam_stats/v_t_mean": 1.5888793102394883e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.8072916865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.1563863456249237,
|
|
"all_logprobs": -0.012487849220633507,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.78125,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.01879574917256832,
|
|
"answer_logprob_reward": 3.049858808517456,
|
|
"answer_logprob_reward/correct": 3.0904715061187744,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.0562736913561821,
|
|
"answer_logprob_reward/incorrect": 2.879723310470581,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.18292562663555145,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 2.079441547393799,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.08679652959108353,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 497.4270935058594,
|
|
"completion_length/correct": 443.2064514160156,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 398.0,
|
|
"completion_length/correct/min": 93.0,
|
|
"completion_length/correct/p25": 293.5,
|
|
"completion_length/correct/p75": 545.0,
|
|
"completion_length/correct/var": 41991.97265625,
|
|
"completion_length/incorrect": 724.5675659179688,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 736.0,
|
|
"completion_length/incorrect/min": 292.0,
|
|
"completion_length/incorrect/p25": 461.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 72786.640625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 431.0,
|
|
"completion_length/min": 93.0,
|
|
"completion_length/p25": 313.0,
|
|
"completion_length/p75": 626.25,
|
|
"completion_length/var": 59956.52734375,
|
|
"epoch": 0.810126582278481,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3402395.0,
|
|
"grad_norm": 0.23521116375923157,
|
|
"kl": 0.013671875,
|
|
"learning_rate": 9.549150281252633e-07,
|
|
"len_reward": 0.09783382713794708,
|
|
"len_reward/correct": 0.16561655700206757,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.20136186480522156,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.004637676291167736,
|
|
"len_reward/correct/p75": 0.3416718542575836,
|
|
"len_reward/correct/var": 0.055385835468769073,
|
|
"len_reward/incorrect": -0.1861209124326706,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.12238359451293945,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3121117949485779,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03858597204089165,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.10651750862598419,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.06163071095943451,
|
|
"len_reward/p75": 0.2982640862464905,
|
|
"len_reward/var": 0.07127738744020462,
|
|
"loss": -0.8067,
|
|
"mean_logprobs": -0.0120849609375,
|
|
"mean_logprobs/var": 0.00015544891357421875,
|
|
"missing_response_penalty": -0.0677083358168602,
|
|
"missing_response_penalty/correct": -0.006451612804085016,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": -1.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.006451612338423729,
|
|
"missing_response_penalty/incorrect": -0.3243243396282196,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.22522523999214172,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.06345440447330475,
|
|
"non_smoothed_logprobs/min": -17.3125,
|
|
"num_completions/total": 6144,
|
|
"policy_loss": -0.8067445755004883,
|
|
"policy_loss/max": 0.0009545485372655094,
|
|
"policy_loss/median": -0.9994170665740967,
|
|
"policy_loss/min": -0.9998272657394409,
|
|
"policy_loss/p25": -0.9995482563972473,
|
|
"policy_loss/p75": -0.9991546869277954,
|
|
"policy_loss/var": 0.15637345612049103,
|
|
"pred_value_log_probs": -0.6289335489273071,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.67578125,
|
|
"pred_value_log_probs/min": -1.2578125,
|
|
"pred_value_log_probs/p25": -0.67578125,
|
|
"pred_value_log_probs/p75": -0.67578125,
|
|
"pred_value_log_probs/var": 0.030305588617920876,
|
|
"reward": 0.8072916865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.1563863456249237,
|
|
"rewards/accuracy_reward": 0.8072916865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.1563863456249237,
|
|
"rewards/answer_logprob_reward": 3.049858808517456,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 2.079441547393799,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.08679652959108353,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09783382713794708,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.10651750862598419,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.06163071095943451,
|
|
"rewards/len_reward/p75": 0.2982640862464905,
|
|
"rewards/len_reward/var": 0.07127738744020462,
|
|
"rewards/missing_response_penalty": -0.0677083358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.06345440447330475,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 32,
|
|
"value_loss": 3.078125,
|
|
"values": -0.7944979071617126,
|
|
"values/max": 2.0,
|
|
"values/median": -0.82080078125,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.82080078125,
|
|
"values/p75": -0.82080078125,
|
|
"values/var": 0.2095223218202591
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 5.642576979880687e-06,
|
|
"adam_stats/lr_effective_mean": 1.6730929142116935e-11,
|
|
"adam_stats/lr_effective_min": -5.89284081797814e-06,
|
|
"adam_stats/m_t_max": 0.0029257573187351227,
|
|
"adam_stats/m_t_mean": 3.1172169880078204e-12,
|
|
"adam_stats/m_t_min": -0.0021233742590993643,
|
|
"adam_stats/v_t_max": 2.3117683667805977e-05,
|
|
"adam_stats/v_t_mean": 1.5888793102394883e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.810126582278481,
|
|
"eval_accuracy_reward": 0.6299999952316284,
|
|
"eval_accuracy_reward/correct": 0.9999999403953552,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23387958109378815,
|
|
"eval_advantages": 0.6299999952316284,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23387958109378815,
|
|
"eval_all_logprobs": -0.013631907291710377,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -6.71875,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02088269777595997,
|
|
"eval_answer_logprob_reward": 3.025315761566162,
|
|
"eval_answer_logprob_reward/correct": 3.106083869934082,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07291579246520996,
|
|
"eval_answer_logprob_reward/incorrect": 2.887791633605957,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.2875722646713257,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1627872735261917,
|
|
"eval_completion_length": 530.8900146484375,
|
|
"eval_completion_length/correct": 459.5661315917969,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 416.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 282.0,
|
|
"eval_completion_length/correct/p75": 607.0,
|
|
"eval_completion_length/correct/var": 49799.45703125,
|
|
"eval_completion_length/incorrect": 652.3333740234375,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 600.0,
|
|
"eval_completion_length/incorrect/min": 160.0,
|
|
"eval_completion_length/incorrect/p25": 399.5,
|
|
"eval_completion_length/incorrect/p75": 997.5,
|
|
"eval_completion_length/incorrect/var": 80186.265625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 493.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 310.75,
|
|
"eval_completion_length/p75": 692.0,
|
|
"eval_completion_length/var": 69502.78125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 5353400.0,
|
|
"eval_len_reward": 0.030771147459745407,
|
|
"eval_len_reward/correct": 0.1357271522283554,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.17058824002742767,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.005235602147877216,
|
|
"eval_len_reward/correct/p75": 0.3363715410232544,
|
|
"eval_len_reward/correct/var": 0.06353338062763214,
|
|
"eval_len_reward/incorrect": -0.14793772995471954,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.0357557013630867,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2836073637008667,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.031824991106987,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.1500728577375412,
|
|
"eval_len_reward/p75": 0.25340428948402405,
|
|
"eval_len_reward/var": 0.07047490775585175,
|
|
"eval_mean_logprobs": -0.01312255859375,
|
|
"eval_mean_logprobs/var": 0.000232696533203125,
|
|
"eval_missing_response_penalty": -0.09333333373069763,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2522522509098053,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -0.5,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.19033581018447876,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.084905244410038,
|
|
"eval_num_completions/total": 9600,
|
|
"eval_pred_value_log_probs": -0.6305449604988098,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6796875,
|
|
"eval_pred_value_log_probs/min": -1.015625,
|
|
"eval_pred_value_log_probs/p25": -0.6796875,
|
|
"eval_pred_value_log_probs/p75": -0.6796875,
|
|
"eval_pred_value_log_probs/var": 0.026422210037708282,
|
|
"eval_reward": 0.6299999952316284,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23387958109378815,
|
|
"eval_rewards/accuracy_reward": 0.6299999952316284,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23387958109378815,
|
|
"eval_rewards/answer_logprob_reward": 3.025315761566162,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1627872735261917,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.030771147459745407,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.1500728577375412,
|
|
"eval_rewards/len_reward/p75": 0.25340428948402405,
|
|
"eval_rewards/len_reward/var": 0.07047490775585175,
|
|
"eval_rewards/missing_response_penalty": -0.09333333373069763,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.084905244410038,
|
|
"eval_runtime": 179.956,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8578459024429321,
|
|
"eval_values/max": 1.9912567138671875,
|
|
"eval_values/median": -0.8193359375,
|
|
"eval_values/min": -1.997598648071289,
|
|
"eval_values/p25": -0.8193359375,
|
|
"eval_values/p75": -0.8193359375,
|
|
"eval_values/var": 0.12734489142894745,
|
|
"step": 32
|
|
},
|
|
{
|
|
"accuracy_reward": 0.71875,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.20320680737495422,
|
|
"adam_stats/lr_effective_max": 4.1723183130670805e-06,
|
|
"adam_stats/lr_effective_mean": 1.9211961882481532e-11,
|
|
"adam_stats/lr_effective_min": -4.256146894476842e-06,
|
|
"adam_stats/m_t_max": 0.003475466975942254,
|
|
"adam_stats/m_t_mean": 3.5379600724777527e-12,
|
|
"adam_stats/m_t_min": -0.002588527277112007,
|
|
"adam_stats/v_t_max": 2.316551035619341e-05,
|
|
"adam_stats/v_t_mean": 1.5930556570079024e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.71875,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.20320680737495422,
|
|
"all_logprobs": -0.012182705104351044,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.0,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.016968917101621628,
|
|
"answer_logprob_reward": 2.9920966625213623,
|
|
"answer_logprob_reward/correct": 3.078829050064087,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06911642849445343,
|
|
"answer_logprob_reward/incorrect": 2.770446300506592,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.3613618314266205,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.16917377710342407,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 554.1458740234375,
|
|
"completion_length/correct": 465.58697509765625,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 394.0,
|
|
"completion_length/correct/min": 137.0,
|
|
"completion_length/correct/p25": 295.0,
|
|
"completion_length/correct/p75": 613.25,
|
|
"completion_length/correct/var": 47973.8515625,
|
|
"completion_length/incorrect": 780.4629516601562,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 768.0,
|
|
"completion_length/incorrect/min": 247.0,
|
|
"completion_length/incorrect/p25": 635.25,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 51827.8046875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 495.0,
|
|
"completion_length/min": 137.0,
|
|
"completion_length/p25": 341.75,
|
|
"completion_length/p75": 754.75,
|
|
"completion_length/var": 68939.421875,
|
|
"epoch": 0.8354430379746836,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3508791.0,
|
|
"grad_norm": 0.22134709358215332,
|
|
"kl": 0.0133056640625,
|
|
"learning_rate": 7.077560319906696e-07,
|
|
"len_reward": 0.08945988863706589,
|
|
"len_reward/correct": 0.19692103564739227,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.24103586375713348,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.011464161798357964,
|
|
"len_reward/correct/p75": 0.3868865966796875,
|
|
"len_reward/correct/var": 0.053773920983076096,
|
|
"len_reward/incorrect": -0.18516303598880768,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.17213115096092224,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.29456648230552673,
|
|
"len_reward/incorrect/p75": -0.004050925839692354,
|
|
"len_reward/incorrect/var": 0.0294047724455595,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.06662239879369736,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.1273576319217682,
|
|
"len_reward/p75": 0.3320743441581726,
|
|
"len_reward/var": 0.07639607042074203,
|
|
"loss": -0.7182,
|
|
"mean_logprobs": -0.01214599609375,
|
|
"mean_logprobs/var": 8.153915405273438e-05,
|
|
"missing_response_penalty": -0.0989583358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.35185185074806213,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.232354998588562,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.08963241428136826,
|
|
"non_smoothed_logprobs/min": -17.4375,
|
|
"num_completions/total": 6336,
|
|
"policy_loss": -0.7182163000106812,
|
|
"policy_loss/max": 0.001128749456256628,
|
|
"policy_loss/median": -0.9993742108345032,
|
|
"policy_loss/min": -0.999798059463501,
|
|
"policy_loss/p25": -0.9995559453964233,
|
|
"policy_loss/p75": 0.0002911695628426969,
|
|
"policy_loss/var": 0.20319871604442596,
|
|
"pred_value_log_probs": -0.6311730742454529,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6796875,
|
|
"pred_value_log_probs/min": -0.7265625,
|
|
"pred_value_log_probs/p25": -0.6796875,
|
|
"pred_value_log_probs/p75": -0.6796875,
|
|
"pred_value_log_probs/var": 0.0247678030282259,
|
|
"reward": 0.71875,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.20320680737495422,
|
|
"rewards/accuracy_reward": 0.71875,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.20320680737495422,
|
|
"rewards/answer_logprob_reward": 2.9920966625213623,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.16917377710342407,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.08945988863706589,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.06662239879369736,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.1273576319217682,
|
|
"rewards/len_reward/p75": 0.3320743441581726,
|
|
"rewards/len_reward/var": 0.07639607042074203,
|
|
"rewards/missing_response_penalty": -0.0989583358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.08963241428136826,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 33,
|
|
"value_loss": 2.8046875,
|
|
"values": -0.8610403537750244,
|
|
"values/max": 1.9999579191207886,
|
|
"values/median": -0.8193359375,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.8193359375,
|
|
"values/p75": -0.8193359375,
|
|
"values/var": 0.1409376859664917
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 4.1723183130670805e-06,
|
|
"adam_stats/lr_effective_mean": 1.9211961882481532e-11,
|
|
"adam_stats/lr_effective_min": -4.256146894476842e-06,
|
|
"adam_stats/m_t_max": 0.003475466975942254,
|
|
"adam_stats/m_t_mean": 3.5379600724777527e-12,
|
|
"adam_stats/m_t_min": -0.002588527277112007,
|
|
"adam_stats/v_t_max": 2.316551035619341e-05,
|
|
"adam_stats/v_t_mean": 1.5930556570079024e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.8354430379746836,
|
|
"eval_accuracy_reward": 0.6133333444595337,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2379487156867981,
|
|
"eval_advantages": 0.6133333444595337,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2379487156867981,
|
|
"eval_all_logprobs": -0.014189048670232296,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.25,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.023575305938720703,
|
|
"eval_answer_logprob_reward": 3.053041696548462,
|
|
"eval_answer_logprob_reward/correct": 3.1240580081939697,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06191055476665497,
|
|
"eval_answer_logprob_reward/incorrect": 2.940394878387451,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.20025868713855743,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.12294083833694458,
|
|
"eval_completion_length": 526.8699951171875,
|
|
"eval_completion_length/correct": 460.2880554199219,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 415.0,
|
|
"eval_completion_length/correct/min": 123.0,
|
|
"eval_completion_length/correct/p25": 288.0,
|
|
"eval_completion_length/correct/p75": 585.5,
|
|
"eval_completion_length/correct/var": 46318.1328125,
|
|
"eval_completion_length/incorrect": 632.4827270507812,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 580.0,
|
|
"eval_completion_length/incorrect/min": 161.0,
|
|
"eval_completion_length/incorrect/p25": 385.25,
|
|
"eval_completion_length/incorrect/p75": 1000.75,
|
|
"eval_completion_length/incorrect/var": 84424.96875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 473.0,
|
|
"eval_completion_length/min": 123.0,
|
|
"eval_completion_length/p25": 311.0,
|
|
"eval_completion_length/p75": 680.25,
|
|
"eval_completion_length/var": 67875.1171875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 5511461.0,
|
|
"eval_len_reward": 0.048468347638845444,
|
|
"eval_len_reward/correct": 0.16129276156425476,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.18827585875988007,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.043197937309741974,
|
|
"eval_len_reward/correct/p75": 0.3443061411380768,
|
|
"eval_len_reward/correct/var": 0.053549688309431076,
|
|
"eval_len_reward/incorrect": -0.13049453496932983,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.011725585907697678,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.260113000869751,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03062245063483715,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.08182896673679352,
|
|
"eval_len_reward/p75": 0.23203158378601074,
|
|
"eval_len_reward/var": 0.0648113414645195,
|
|
"eval_mean_logprobs": -0.0137939453125,
|
|
"eval_mean_logprobs/var": 0.00011444091796875,
|
|
"eval_missing_response_penalty": -0.08666666597127914,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.22413793206214905,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.17541228234767914,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0794202908873558,
|
|
"eval_num_completions/total": 9900,
|
|
"eval_pred_value_log_probs": -0.6236971020698547,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.671875,
|
|
"eval_pred_value_log_probs/min": -0.95703125,
|
|
"eval_pred_value_log_probs/p25": -0.671875,
|
|
"eval_pred_value_log_probs/p75": -0.671875,
|
|
"eval_pred_value_log_probs/var": 0.02542439103126526,
|
|
"eval_reward": 0.6133333444595337,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2379487156867981,
|
|
"eval_rewards/accuracy_reward": 0.6133333444595337,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2379487156867981,
|
|
"eval_rewards/answer_logprob_reward": 3.053041696548462,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.12294083833694458,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.048468347638845444,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.08182896673679352,
|
|
"eval_rewards/len_reward/p75": 0.23203158378601074,
|
|
"eval_rewards/len_reward/var": 0.0648113414645195,
|
|
"eval_rewards/missing_response_penalty": -0.08666666597127914,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0794202908873558,
|
|
"eval_runtime": 167.6439,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8649055361747742,
|
|
"eval_values/max": 1.9913864135742188,
|
|
"eval_values/median": -0.82763671875,
|
|
"eval_values/min": -1.9897403717041016,
|
|
"eval_values/p25": -0.82763671875,
|
|
"eval_values/p75": -0.82763671875,
|
|
"eval_values/var": 0.12723535299301147,
|
|
"step": 33
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7552083730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.18583661317825317,
|
|
"adam_stats/lr_effective_max": 2.912940090027405e-06,
|
|
"adam_stats/lr_effective_mean": 1.2614843197811254e-11,
|
|
"adam_stats/lr_effective_min": -2.9891004942328436e-06,
|
|
"adam_stats/m_t_max": 0.0034666655119508505,
|
|
"adam_stats/m_t_mean": 8.689088511204535e-12,
|
|
"adam_stats/m_t_min": -0.0025768668856471777,
|
|
"adam_stats/v_t_max": 2.315381971129682e-05,
|
|
"adam_stats/v_t_mean": 1.5974886344305439e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7552083730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.18583661317825317,
|
|
"all_logprobs": -0.013020492158830166,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -7.15625,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.021044790744781494,
|
|
"answer_logprob_reward": 3.0814647674560547,
|
|
"answer_logprob_reward/correct": 3.120690107345581,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.036724288016557693,
|
|
"answer_logprob_reward/incorrect": 2.960451364517212,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.13478022813796997,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 2.079441547393799,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.06491918861865997,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 488.953125,
|
|
"completion_length/correct": 431.220703125,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 385.0,
|
|
"completion_length/correct/min": 154.0,
|
|
"completion_length/correct/p25": 284.0,
|
|
"completion_length/correct/p75": 543.0,
|
|
"completion_length/correct/var": 36086.546875,
|
|
"completion_length/incorrect": 667.0637817382812,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 603.0,
|
|
"completion_length/incorrect/min": 223.0,
|
|
"completion_length/incorrect/p25": 470.5,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 70686.1015625,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 434.0,
|
|
"completion_length/min": 154.0,
|
|
"completion_length/p25": 305.75,
|
|
"completion_length/p75": 610.0,
|
|
"completion_length/var": 54567.08984375,
|
|
"epoch": 0.8607594936708861,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3602670.0,
|
|
"grad_norm": 0.22213982045650482,
|
|
"kl": 0.013824462890625,
|
|
"learning_rate": 4.951556604879049e-07,
|
|
"len_reward": 0.1257983297109604,
|
|
"len_reward/correct": 0.2184109091758728,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.26282352209091187,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.08117646723985672,
|
|
"len_reward/correct/p75": 0.38985350728034973,
|
|
"len_reward/correct/var": 0.04815307632088661,
|
|
"len_reward/incorrect": -0.1599213033914566,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.004919730592519045,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.37439972162246704,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.04051285609602928,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.15333938598632812,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.002818167908117175,
|
|
"len_reward/p75": 0.349761962890625,
|
|
"len_reward/var": 0.07266069203615189,
|
|
"loss": -0.7547,
|
|
"mean_logprobs": -0.012451171875,
|
|
"mean_logprobs/var": 0.0001010894775390625,
|
|
"missing_response_penalty": -0.0625,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.25531914830207825,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -0.5,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.19426457583904266,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.05890052020549774,
|
|
"non_smoothed_logprobs/min": -17.25,
|
|
"num_completions/total": 6528,
|
|
"policy_loss": -0.7546552419662476,
|
|
"policy_loss/max": 0.0010960586369037628,
|
|
"policy_loss/median": -0.9994020462036133,
|
|
"policy_loss/min": -0.9998460412025452,
|
|
"policy_loss/p25": -0.9995385408401489,
|
|
"policy_loss/p75": -0.9988569617271423,
|
|
"policy_loss/var": 0.18583497405052185,
|
|
"pred_value_log_probs": -0.6407127380371094,
|
|
"pred_value_log_probs/max": -0.03466796875,
|
|
"pred_value_log_probs/median": -0.671875,
|
|
"pred_value_log_probs/min": -1.3046875,
|
|
"pred_value_log_probs/p25": -0.671875,
|
|
"pred_value_log_probs/p75": -0.671875,
|
|
"pred_value_log_probs/var": 0.025662986561655998,
|
|
"reward": 0.7552083730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.18583661317825317,
|
|
"rewards/accuracy_reward": 0.7552083730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.18583661317825317,
|
|
"rewards/answer_logprob_reward": 3.0814647674560547,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 2.079441547393799,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.06491918861865997,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.1257983297109604,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.15333938598632812,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.002818167908117175,
|
|
"rewards/len_reward/p75": 0.349761962890625,
|
|
"rewards/len_reward/var": 0.07266069203615189,
|
|
"rewards/missing_response_penalty": -0.0625,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.05890052020549774,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 34,
|
|
"value_loss": 2.6796875,
|
|
"values": -0.8266692161560059,
|
|
"values/max": 1.817474365234375,
|
|
"values/median": -0.82763671875,
|
|
"values/min": -1.901123046875,
|
|
"values/p25": -0.82763671875,
|
|
"values/p75": -0.82763671875,
|
|
"values/var": 0.16586531698703766
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 2.912940090027405e-06,
|
|
"adam_stats/lr_effective_mean": 1.2614843197811254e-11,
|
|
"adam_stats/lr_effective_min": -2.9891004942328436e-06,
|
|
"adam_stats/m_t_max": 0.0034666655119508505,
|
|
"adam_stats/m_t_mean": 8.689088511204535e-12,
|
|
"adam_stats/m_t_min": -0.0025768668856471777,
|
|
"adam_stats/v_t_max": 2.315381971129682e-05,
|
|
"adam_stats/v_t_mean": 1.5974886344305439e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.8607594936708861,
|
|
"eval_accuracy_reward": 0.6066666841506958,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23942029476165771,
|
|
"eval_advantages": 0.6066666841506958,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23942029476165771,
|
|
"eval_all_logprobs": -0.013087671250104904,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.25,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.019924871623516083,
|
|
"eval_answer_logprob_reward": 3.0365922451019287,
|
|
"eval_answer_logprob_reward/correct": 3.110943555831909,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.0733567550778389,
|
|
"eval_answer_logprob_reward/incorrect": 2.921915292739868,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.24640268087387085,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1493799239397049,
|
|
"eval_completion_length": 528.7733764648438,
|
|
"eval_completion_length/correct": 449.76373291015625,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 423.0,
|
|
"eval_completion_length/correct/min": 102.0,
|
|
"eval_completion_length/correct/p25": 292.75,
|
|
"eval_completion_length/correct/p75": 575.5,
|
|
"eval_completion_length/correct/var": 39934.58984375,
|
|
"eval_completion_length/incorrect": 650.6356201171875,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 605.0,
|
|
"eval_completion_length/incorrect/min": 161.0,
|
|
"eval_completion_length/incorrect/p25": 383.0,
|
|
"eval_completion_length/incorrect/p75": 1011.75,
|
|
"eval_completion_length/incorrect/var": 88653.1640625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 458.0,
|
|
"eval_completion_length/min": 102.0,
|
|
"eval_completion_length/p25": 317.75,
|
|
"eval_completion_length/p75": 680.5,
|
|
"eval_completion_length/var": 68525.3046875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 5670093.0,
|
|
"eval_len_reward": 0.033590104430913925,
|
|
"eval_len_reward/correct": 0.14972682297229767,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.164000004529953,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.011975601315498352,
|
|
"eval_len_reward/correct/p75": 0.31185182929039,
|
|
"eval_len_reward/correct/var": 0.05152174457907677,
|
|
"eval_len_reward/incorrect": -0.14553600549697876,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.03591160103678703,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.30224093794822693,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03321918472647667,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.09572094678878784,
|
|
"eval_len_reward/p75": 0.22523236274719238,
|
|
"eval_len_reward/var": 0.06506025791168213,
|
|
"eval_mean_logprobs": -0.012451171875,
|
|
"eval_mean_logprobs/var": 8.058547973632812e-05,
|
|
"eval_missing_response_penalty": -0.09333333373069763,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.23728813230991364,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.18252931535243988,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.084905244410038,
|
|
"eval_num_completions/total": 10200,
|
|
"eval_pred_value_log_probs": -0.6302194595336914,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6796875,
|
|
"eval_pred_value_log_probs/min": -0.97265625,
|
|
"eval_pred_value_log_probs/p25": -0.6796875,
|
|
"eval_pred_value_log_probs/p75": -0.6796875,
|
|
"eval_pred_value_log_probs/var": 0.026019588112831116,
|
|
"eval_reward": 0.6066666841506958,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23942029476165771,
|
|
"eval_rewards/accuracy_reward": 0.6066666841506958,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23942029476165771,
|
|
"eval_rewards/answer_logprob_reward": 3.0365922451019287,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1493799239397049,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.033590104430913925,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.09572094678878784,
|
|
"eval_rewards/len_reward/p75": 0.22523236274719238,
|
|
"eval_rewards/len_reward/var": 0.06506025791168213,
|
|
"eval_rewards/missing_response_penalty": -0.09333333373069763,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.084905244410038,
|
|
"eval_runtime": 172.9978,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8618219494819641,
|
|
"eval_values/max": 1.9911537170410156,
|
|
"eval_values/median": -0.8232421875,
|
|
"eval_values/min": -1.9976177215576172,
|
|
"eval_values/p25": -0.8232421875,
|
|
"eval_values/p75": -0.8232421875,
|
|
"eval_values/var": 0.12568846344947815,
|
|
"step": 34
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7395833730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.19360822439193726,
|
|
"adam_stats/lr_effective_max": 1.8681564597500255e-06,
|
|
"adam_stats/lr_effective_mean": 3.9044540170452e-12,
|
|
"adam_stats/lr_effective_min": -1.9609640276030404e-06,
|
|
"adam_stats/m_t_max": 0.003876834874972701,
|
|
"adam_stats/m_t_mean": -3.680513793041795e-12,
|
|
"adam_stats/m_t_min": -0.0027922026347368956,
|
|
"adam_stats/v_t_max": 2.3187945771496743e-05,
|
|
"adam_stats/v_t_mean": 1.6035186416532565e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7395833730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.19360822439193726,
|
|
"all_logprobs": -0.011429689824581146,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -5.875,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.016206832602620125,
|
|
"answer_logprob_reward": 3.0354182720184326,
|
|
"answer_logprob_reward/correct": 3.082453489303589,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06069010868668556,
|
|
"answer_logprob_reward/incorrect": 2.901837110519409,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.16296735405921936,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 2.079441547393799,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.09292694181203842,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 535.640625,
|
|
"completion_length/correct": 487.45068359375,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 457.0,
|
|
"completion_length/correct/min": 95.0,
|
|
"completion_length/correct/p25": 341.5,
|
|
"completion_length/correct/p75": 614.5,
|
|
"completion_length/correct/var": 44771.44140625,
|
|
"completion_length/incorrect": 672.5,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 637.0,
|
|
"completion_length/incorrect/min": 205.0,
|
|
"completion_length/incorrect/p25": 426.25,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 80784.578125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 470.0,
|
|
"completion_length/min": 95.0,
|
|
"completion_length/p25": 363.0,
|
|
"completion_length/p75": 668.5,
|
|
"completion_length/var": 60405.78515625,
|
|
"epoch": 0.8860759493670886,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3705513.0,
|
|
"grad_norm": 0.23321214318275452,
|
|
"kl": 0.013275146484375,
|
|
"learning_rate": 3.18825646801314e-07,
|
|
"len_reward": 0.041230458766222,
|
|
"len_reward/correct": 0.11422709375619888,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.10663507133722305,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": -0.045150503516197205,
|
|
"len_reward/correct/p75": 0.291431725025177,
|
|
"len_reward/correct/var": 0.05920525640249252,
|
|
"len_reward/incorrect": -0.16608001291751862,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.14477798342704773,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.2727471590042114,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03390161693096161,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.0039627039805054665,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.1009591817855835,
|
|
"len_reward/p75": 0.21456876397132874,
|
|
"len_reward/var": 0.06761597096920013,
|
|
"loss": -0.7391,
|
|
"mean_logprobs": -0.011962890625,
|
|
"mean_logprobs/var": 8.392333984375e-05,
|
|
"missing_response_penalty": -0.0677083358168602,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.25999999046325684,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -0.75,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.19632652401924133,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.06345440447330475,
|
|
"non_smoothed_logprobs/min": -17.125,
|
|
"num_completions/total": 6720,
|
|
"policy_loss": -0.7390528917312622,
|
|
"policy_loss/max": 0.0013184287818148732,
|
|
"policy_loss/median": -0.9994184970855713,
|
|
"policy_loss/min": -0.999813973903656,
|
|
"policy_loss/p25": -0.9995690584182739,
|
|
"policy_loss/p75": 0.0002589412033557892,
|
|
"policy_loss/var": 0.19361191987991333,
|
|
"pred_value_log_probs": -0.6269299983978271,
|
|
"pred_value_log_probs/max": -1.0728836059570312e-06,
|
|
"pred_value_log_probs/median": -0.6796875,
|
|
"pred_value_log_probs/min": -1.203125,
|
|
"pred_value_log_probs/p25": -0.6796875,
|
|
"pred_value_log_probs/p75": -0.6796875,
|
|
"pred_value_log_probs/var": 0.03591961786150932,
|
|
"reward": 0.7395833730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.19360822439193726,
|
|
"rewards/accuracy_reward": 0.7395833730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.19360822439193726,
|
|
"rewards/answer_logprob_reward": 3.0354182720184326,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 2.079441547393799,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.09292694181203842,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.041230458766222,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.0039627039805054665,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.1009591817855835,
|
|
"rewards/len_reward/p75": 0.21456876397132874,
|
|
"rewards/len_reward/var": 0.06761597096920013,
|
|
"rewards/missing_response_penalty": -0.0677083358168602,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.06345440447330475,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 35,
|
|
"value_loss": 2.78125,
|
|
"values": -0.8612565994262695,
|
|
"values/max": 1.55291748046875,
|
|
"values/median": -0.8232421875,
|
|
"values/min": -1.9980392456054688,
|
|
"values/p25": -0.8232421875,
|
|
"values/p75": -0.8232421875,
|
|
"values/var": 0.179811492562294
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 1.8681564597500255e-06,
|
|
"adam_stats/lr_effective_mean": 3.9044540170452e-12,
|
|
"adam_stats/lr_effective_min": -1.9609640276030404e-06,
|
|
"adam_stats/m_t_max": 0.003876834874972701,
|
|
"adam_stats/m_t_mean": -3.680513793041795e-12,
|
|
"adam_stats/m_t_min": -0.0027922026347368956,
|
|
"adam_stats/v_t_max": 2.3187945771496743e-05,
|
|
"adam_stats/v_t_mean": 1.6035186416532565e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.8860759493670886,
|
|
"eval_accuracy_reward": 0.6366666555404663,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.2320958822965622,
|
|
"eval_advantages": 0.6366666555404663,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.2320958822965622,
|
|
"eval_all_logprobs": -0.01600293070077896,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -7.0,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02648807317018509,
|
|
"eval_answer_logprob_reward": 3.058274030685425,
|
|
"eval_answer_logprob_reward/correct": 3.107264995574951,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.07228077203035355,
|
|
"eval_answer_logprob_reward/incorrect": 2.9724276065826416,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/incorrect/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.230248361825943,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.13331730663776398,
|
|
"eval_completion_length": 524.0466918945312,
|
|
"eval_completion_length/correct": 463.9371643066406,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 427.0,
|
|
"eval_completion_length/correct/min": 113.0,
|
|
"eval_completion_length/correct/p25": 296.0,
|
|
"eval_completion_length/correct/p75": 610.5,
|
|
"eval_completion_length/correct/var": 44254.5,
|
|
"eval_completion_length/incorrect": 629.3760986328125,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 588.0,
|
|
"eval_completion_length/incorrect/min": 160.0,
|
|
"eval_completion_length/incorrect/p25": 360.0,
|
|
"eval_completion_length/incorrect/p75": 975.0,
|
|
"eval_completion_length/incorrect/var": 86309.28125,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 466.0,
|
|
"eval_completion_length/min": 113.0,
|
|
"eval_completion_length/p25": 311.75,
|
|
"eval_completion_length/p75": 679.75,
|
|
"eval_completion_length/var": 65649.328125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 5827307.0,
|
|
"eval_len_reward": 0.04057655483484268,
|
|
"eval_len_reward/correct": 0.14831168949604034,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.18240000307559967,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.0,
|
|
"eval_len_reward/correct/p75": 0.310127854347229,
|
|
"eval_len_reward/correct/var": 0.05233708769083023,
|
|
"eval_len_reward/incorrect": -0.14820705354213715,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.010466397739946842,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.30244675278663635,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.035756561905145645,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.10545950382947922,
|
|
"eval_len_reward/p75": 0.23584264516830444,
|
|
"eval_len_reward/var": 0.06657975167036057,
|
|
"eval_mean_logprobs": -0.0145263671875,
|
|
"eval_mean_logprobs/var": 0.00049591064453125,
|
|
"eval_missing_response_penalty": -0.07999999821186066,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.22018347680568695,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.17329256236553192,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07384615391492844,
|
|
"eval_num_completions/total": 10500,
|
|
"eval_pred_value_log_probs": -0.6315984129905701,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6796875,
|
|
"eval_pred_value_log_probs/min": -1.0703125,
|
|
"eval_pred_value_log_probs/p25": -0.6796875,
|
|
"eval_pred_value_log_probs/p75": -0.6796875,
|
|
"eval_pred_value_log_probs/var": 0.025896800681948662,
|
|
"eval_reward": 0.6366666555404663,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.2320958822965622,
|
|
"eval_rewards/accuracy_reward": 0.6366666555404663,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.2320958822965622,
|
|
"eval_rewards/answer_logprob_reward": 3.058274030685425,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.13331730663776398,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.04057655483484268,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.10545950382947922,
|
|
"eval_rewards/len_reward/p75": 0.23584264516830444,
|
|
"eval_rewards/len_reward/var": 0.06657975167036057,
|
|
"eval_rewards/missing_response_penalty": -0.07999999821186066,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07384615391492844,
|
|
"eval_runtime": 166.2341,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8527399301528931,
|
|
"eval_values/max": 1.9915237426757812,
|
|
"eval_values/median": -0.8154296875,
|
|
"eval_values/min": -1.9976787567138672,
|
|
"eval_values/p25": -0.8154296875,
|
|
"eval_values/p75": -0.8154296875,
|
|
"eval_values/var": 0.12768949568271637,
|
|
"step": 35
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7864583730697632,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.16882088780403137,
|
|
"adam_stats/lr_effective_max": 1.0349542662879685e-06,
|
|
"adam_stats/lr_effective_mean": -5.445842344783958e-13,
|
|
"adam_stats/lr_effective_min": -1.0831272447830997e-06,
|
|
"adam_stats/m_t_max": 0.003202286083251238,
|
|
"adam_stats/m_t_mean": -3.4583507932395285e-12,
|
|
"adam_stats/m_t_min": -0.0023229410871863365,
|
|
"adam_stats/v_t_max": 2.3172986402641982e-05,
|
|
"adam_stats/v_t_mean": 1.6057590370224806e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7864583730697632,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.16882088780403137,
|
|
"all_logprobs": -0.01194229256361723,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -6.96875,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.018675291910767555,
|
|
"answer_logprob_reward": 2.993354558944702,
|
|
"answer_logprob_reward/correct": 3.0919620990753174,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.06152000278234482,
|
|
"answer_logprob_reward/incorrect": 2.630190849304199,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.2321436405181885,
|
|
"answer_logprob_reward/incorrect/p25": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.6376937627792358,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.2321436405181885,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.21786069869995117,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 525.8958740234375,
|
|
"completion_length/correct": 454.3907165527344,
|
|
"completion_length/correct/max": 1024.0,
|
|
"completion_length/correct/median": 421.0,
|
|
"completion_length/correct/min": 140.0,
|
|
"completion_length/correct/p25": 311.5,
|
|
"completion_length/correct/p75": 542.5,
|
|
"completion_length/correct/var": 38425.33203125,
|
|
"completion_length/incorrect": 789.243896484375,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 932.0,
|
|
"completion_length/incorrect/min": 251.0,
|
|
"completion_length/incorrect/p25": 568.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 72143.4921875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 466.0,
|
|
"completion_length/min": 140.0,
|
|
"completion_length/p25": 331.75,
|
|
"completion_length/p75": 669.75,
|
|
"completion_length/var": 64214.8671875,
|
|
"epoch": 0.9113924050632911,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3806485.0,
|
|
"grad_norm": 0.22095003724098206,
|
|
"kl": 0.013275146484375,
|
|
"learning_rate": 1.801856965207338e-07,
|
|
"len_reward": 0.09725229442119598,
|
|
"len_reward/correct": 0.17949548363685608,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2398190051317215,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.04010206088423729,
|
|
"len_reward/correct/p75": 0.3279731869697571,
|
|
"len_reward/correct/var": 0.0479915477335453,
|
|
"len_reward/incorrect": -0.205643430352211,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1445556879043579,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.35064664483070374,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.037356629967689514,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.10348469018936157,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.04575807601213455,
|
|
"len_reward/p75": 0.31137803196907043,
|
|
"len_reward/var": 0.07055462151765823,
|
|
"loss": -0.7859,
|
|
"mean_logprobs": -0.0115966796875,
|
|
"mean_logprobs/var": 7.343292236328125e-05,
|
|
"missing_response_penalty": -0.0885416716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4146341383457184,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2487805187702179,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.08112456649541855,
|
|
"non_smoothed_logprobs/min": -17.25,
|
|
"num_completions/total": 6912,
|
|
"policy_loss": -0.7859267592430115,
|
|
"policy_loss/max": 0.0006974264979362488,
|
|
"policy_loss/median": -0.9994094371795654,
|
|
"policy_loss/min": -0.9998270869255066,
|
|
"policy_loss/p25": -0.9995585680007935,
|
|
"policy_loss/p75": -0.9990115165710449,
|
|
"policy_loss/var": 0.16880235075950623,
|
|
"pred_value_log_probs": -0.6317064166069031,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6796875,
|
|
"pred_value_log_probs/min": -1.0703125,
|
|
"pred_value_log_probs/p25": -0.6796875,
|
|
"pred_value_log_probs/p75": -0.6796875,
|
|
"pred_value_log_probs/var": 0.031910497695207596,
|
|
"reward": 0.7864583730697632,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.16882088780403137,
|
|
"rewards/accuracy_reward": 0.7864583730697632,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.16882088780403137,
|
|
"rewards/answer_logprob_reward": 2.993354558944702,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.2321436405181885,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.21786069869995117,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09725229442119598,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.10348469018936157,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.04575807601213455,
|
|
"rewards/len_reward/p75": 0.31137803196907043,
|
|
"rewards/len_reward/var": 0.07055462151765823,
|
|
"rewards/missing_response_penalty": -0.0885416716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.08112456649541855,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 36,
|
|
"value_loss": 3.0,
|
|
"values": -0.8033027648925781,
|
|
"values/max": 1.9813604354858398,
|
|
"values/median": -0.8154296875,
|
|
"values/min": -1.9923381805419922,
|
|
"values/p25": -0.8154296875,
|
|
"values/p75": -0.8154296875,
|
|
"values/var": 0.20548765361309052
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 1.0349542662879685e-06,
|
|
"adam_stats/lr_effective_mean": -5.445842344783958e-13,
|
|
"adam_stats/lr_effective_min": -1.0831272447830997e-06,
|
|
"adam_stats/m_t_max": 0.003202286083251238,
|
|
"adam_stats/m_t_mean": -3.4583507932395285e-12,
|
|
"adam_stats/m_t_min": -0.0023229410871863365,
|
|
"adam_stats/v_t_max": 2.3172986402641982e-05,
|
|
"adam_stats/v_t_mean": 1.6057590370224806e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.9113924050632911,
|
|
"eval_accuracy_reward": 0.6033333539962769,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.24012264609336853,
|
|
"eval_advantages": 0.6033333539962769,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.24012264609336853,
|
|
"eval_all_logprobs": -0.013793468475341797,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -6.9375,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02206967957317829,
|
|
"eval_answer_logprob_reward": 3.00046706199646,
|
|
"eval_answer_logprob_reward/correct": 3.1109981536865234,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06843782961368561,
|
|
"eval_answer_logprob_reward/incorrect": 2.832347869873047,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.35508111119270325,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1999768763780594,
|
|
"eval_completion_length": 539.5366821289062,
|
|
"eval_completion_length/correct": 459.8840026855469,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 405.0,
|
|
"eval_completion_length/correct/min": 123.0,
|
|
"eval_completion_length/correct/p25": 284.0,
|
|
"eval_completion_length/correct/p75": 585.0,
|
|
"eval_completion_length/correct/var": 46491.90234375,
|
|
"eval_completion_length/incorrect": 660.6890869140625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 639.0,
|
|
"eval_completion_length/incorrect/min": 161.0,
|
|
"eval_completion_length/incorrect/p25": 404.0,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 86875.921875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 476.0,
|
|
"eval_completion_length/min": 123.0,
|
|
"eval_completion_length/p25": 311.75,
|
|
"eval_completion_length/p75": 729.75,
|
|
"eval_completion_length/var": 71956.3125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 5989168.0,
|
|
"eval_len_reward": 0.06377188116312027,
|
|
"eval_len_reward/correct": 0.18652506172657013,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.2197403609752655,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.040242165327072144,
|
|
"eval_len_reward/correct/p75": 0.3611909747123718,
|
|
"eval_len_reward/correct/var": 0.052537575364112854,
|
|
"eval_len_reward/incorrect": -0.12293671816587448,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": 0.0,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.24470418691635132,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.029403019696474075,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.06674864888191223,
|
|
"eval_len_reward/p75": 0.26565515995025635,
|
|
"eval_len_reward/var": 0.06622755527496338,
|
|
"eval_mean_logprobs": -0.01300048828125,
|
|
"eval_mean_logprobs/var": 0.00014495849609375,
|
|
"eval_missing_response_penalty": -0.1133333370089531,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2857142984867096,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.20581114292144775,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.10082497447729111,
|
|
"eval_num_completions/total": 10800,
|
|
"eval_pred_value_log_probs": -0.6211370229721069,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.66796875,
|
|
"eval_pred_value_log_probs/min": -1.015625,
|
|
"eval_pred_value_log_probs/p25": -0.66796875,
|
|
"eval_pred_value_log_probs/p75": -0.66796875,
|
|
"eval_pred_value_log_probs/var": 0.025079844519495964,
|
|
"eval_reward": 0.6033333539962769,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.24012264609336853,
|
|
"eval_rewards/accuracy_reward": 0.6033333539962769,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.24012264609336853,
|
|
"eval_rewards/answer_logprob_reward": 3.00046706199646,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1999768763780594,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.06377188116312027,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.06674864888191223,
|
|
"eval_rewards/len_reward/p75": 0.26565515995025635,
|
|
"eval_rewards/len_reward/var": 0.06622755527496338,
|
|
"eval_rewards/missing_response_penalty": -0.1133333370089531,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.10082497447729111,
|
|
"eval_runtime": 166.1014,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8637380003929138,
|
|
"eval_values/max": 1.991241455078125,
|
|
"eval_values/median": -0.82666015625,
|
|
"eval_values/min": -1.9975910186767578,
|
|
"eval_values/p25": -0.82666015625,
|
|
"eval_values/p75": -0.82666015625,
|
|
"eval_values/var": 0.12652793526649475,
|
|
"step": 36
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7291666865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.19851656258106232,
|
|
"adam_stats/lr_effective_max": 4.674313629493554e-07,
|
|
"adam_stats/lr_effective_mean": -8.888273147003578e-13,
|
|
"adam_stats/lr_effective_min": -4.590939965964935e-07,
|
|
"adam_stats/m_t_max": 0.0032116472721099854,
|
|
"adam_stats/m_t_mean": -1.6246075665304893e-11,
|
|
"adam_stats/m_t_min": -0.0027345679700374603,
|
|
"adam_stats/v_t_max": 2.3160677301348187e-05,
|
|
"adam_stats/v_t_mean": 1.6107731468095743e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7291666865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.19851656258106232,
|
|
"all_logprobs": -0.014034952968358994,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -8.0,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.021980132907629013,
|
|
"answer_logprob_reward": 2.9622597694396973,
|
|
"answer_logprob_reward/correct": 3.0406382083892822,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.0865296870470047,
|
|
"answer_logprob_reward/incorrect": 2.751241445541382,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.079441547393799,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.2960805594921112,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.15865588188171387,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 531.421875,
|
|
"completion_length/correct": 444.3071594238281,
|
|
"completion_length/correct/max": 1002.0,
|
|
"completion_length/correct/median": 411.0,
|
|
"completion_length/correct/min": 170.0,
|
|
"completion_length/correct/p25": 293.75,
|
|
"completion_length/correct/p75": 534.0,
|
|
"completion_length/correct/var": 30305.453125,
|
|
"completion_length/incorrect": 765.9615478515625,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 805.0,
|
|
"completion_length/incorrect/min": 101.0,
|
|
"completion_length/incorrect/p25": 561.75,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 81878.4296875,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 463.0,
|
|
"completion_length/min": 101.0,
|
|
"completion_length/p25": 340.75,
|
|
"completion_length/p75": 668.75,
|
|
"completion_length/var": 64456.4140625,
|
|
"epoch": 0.9367088607594937,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 3908518.0,
|
|
"grad_norm": 0.22948013246059418,
|
|
"kl": 0.0140380859375,
|
|
"learning_rate": 8.035205700685167e-08,
|
|
"len_reward": 0.10592197626829147,
|
|
"len_reward/correct": 0.2098534256219864,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.24943362176418304,
|
|
"len_reward/correct/min": -0.5,
|
|
"len_reward/correct/p25": 0.08449641615152359,
|
|
"len_reward/correct/p75": 0.35881540179252625,
|
|
"len_reward/correct/var": 0.043855007737874985,
|
|
"len_reward/incorrect": -0.1738934963941574,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1606655716896057,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.2525708079338074,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.032235048711299896,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.14386044442653656,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.042322441935539246,
|
|
"len_reward/p75": 0.31434303522109985,
|
|
"len_reward/var": 0.06975657492876053,
|
|
"loss": -0.7286,
|
|
"mean_logprobs": -0.013916015625,
|
|
"mean_logprobs/var": 0.000415802001953125,
|
|
"missing_response_penalty": -0.1041666716337204,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.38461539149284363,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.24132730066776276,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.09380454570055008,
|
|
"non_smoothed_logprobs/min": -17.4375,
|
|
"num_completions/total": 7104,
|
|
"policy_loss": -0.7286052107810974,
|
|
"policy_loss/max": 0.0022027799859642982,
|
|
"policy_loss/median": -0.9993489980697632,
|
|
"policy_loss/min": -0.9998674988746643,
|
|
"policy_loss/p25": -0.9995580315589905,
|
|
"policy_loss/p75": 0.0002452961343806237,
|
|
"policy_loss/var": 0.19849976897239685,
|
|
"pred_value_log_probs": -0.607271134853363,
|
|
"pred_value_log_probs/max": -0.0019378662109375,
|
|
"pred_value_log_probs/median": -0.66796875,
|
|
"pred_value_log_probs/min": -1.2734375,
|
|
"pred_value_log_probs/p25": -0.66796875,
|
|
"pred_value_log_probs/p75": -0.66796875,
|
|
"pred_value_log_probs/var": 0.04597088694572449,
|
|
"reward": 0.7291666865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.19851656258106232,
|
|
"rewards/accuracy_reward": 0.7291666865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.19851656258106232,
|
|
"rewards/answer_logprob_reward": 2.9622597694396973,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.15865588188171387,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.10592197626829147,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.14386044442653656,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.042322441935539246,
|
|
"rewards/len_reward/p75": 0.31434303522109985,
|
|
"rewards/len_reward/var": 0.06975657492876053,
|
|
"rewards/missing_response_penalty": -0.1041666716337204,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.09380454570055008,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 37,
|
|
"value_loss": 2.796875,
|
|
"values": -0.8294498324394226,
|
|
"values/max": 1.11810302734375,
|
|
"values/median": -0.82666015625,
|
|
"values/min": -1.9980120658874512,
|
|
"values/p25": -0.82666015625,
|
|
"values/p75": -0.82666015625,
|
|
"values/var": 0.19865192472934723
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 4.674313629493554e-07,
|
|
"adam_stats/lr_effective_mean": -8.888273147003578e-13,
|
|
"adam_stats/lr_effective_min": -4.590939965964935e-07,
|
|
"adam_stats/m_t_max": 0.0032116472721099854,
|
|
"adam_stats/m_t_mean": -1.6246075665304893e-11,
|
|
"adam_stats/m_t_min": -0.0027345679700374603,
|
|
"adam_stats/v_t_max": 2.3160677301348187e-05,
|
|
"adam_stats/v_t_mean": 1.6107731468095743e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.9367088607594937,
|
|
"eval_accuracy_reward": 0.6266666650772095,
|
|
"eval_accuracy_reward/correct": 0.9999999403953552,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23473802208900452,
|
|
"eval_advantages": 0.6266666650772095,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23473802208900452,
|
|
"eval_all_logprobs": -0.013606792315840721,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -10.0,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.02223222889006138,
|
|
"eval_answer_logprob_reward": 3.021596670150757,
|
|
"eval_answer_logprob_reward/correct": 3.1150147914886475,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06629540771245956,
|
|
"eval_answer_logprob_reward/incorrect": 2.864788293838501,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.2743638753890991,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.1580141931772232,
|
|
"eval_completion_length": 531.4367065429688,
|
|
"eval_completion_length/correct": 460.1861572265625,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 438.0,
|
|
"eval_completion_length/correct/min": 123.0,
|
|
"eval_completion_length/correct/p25": 289.75,
|
|
"eval_completion_length/correct/p75": 577.75,
|
|
"eval_completion_length/correct/var": 42336.1328125,
|
|
"eval_completion_length/incorrect": 651.0357666015625,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 603.0,
|
|
"eval_completion_length/incorrect/min": 160.0,
|
|
"eval_completion_length/incorrect/p25": 393.75,
|
|
"eval_completion_length/incorrect/p75": 1024.0,
|
|
"eval_completion_length/incorrect/var": 83656.65625,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 482.0,
|
|
"eval_completion_length/min": 123.0,
|
|
"eval_completion_length/p25": 312.75,
|
|
"eval_completion_length/p75": 694.0,
|
|
"eval_completion_length/var": 66084.2578125,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 6148599.0,
|
|
"eval_len_reward": 0.03777023404836655,
|
|
"eval_len_reward/correct": 0.14650936424732208,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.16897918283939362,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.005582460202276707,
|
|
"eval_len_reward/correct/p75": 0.31155991554260254,
|
|
"eval_len_reward/correct/var": 0.05335502699017525,
|
|
"eval_len_reward/incorrect": -0.14475619792938232,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": -0.023785926401615143,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.2747696042060852,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.032570742070674896,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.08813288807868958,
|
|
"eval_len_reward/p75": 0.226766437292099,
|
|
"eval_len_reward/var": 0.06537482887506485,
|
|
"eval_mean_logprobs": -0.01318359375,
|
|
"eval_mean_logprobs/var": 0.00010204315185546875,
|
|
"eval_missing_response_penalty": -0.09666667133569717,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.2589285969734192,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.19361327588558197,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0876142680644989,
|
|
"eval_num_completions/total": 11100,
|
|
"eval_pred_value_log_probs": -0.6239250302314758,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.671875,
|
|
"eval_pred_value_log_probs/min": -1.0234375,
|
|
"eval_pred_value_log_probs/p25": -0.671875,
|
|
"eval_pred_value_log_probs/p75": -0.671875,
|
|
"eval_pred_value_log_probs/var": 0.025610415264964104,
|
|
"eval_reward": 0.6266666650772095,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23473802208900452,
|
|
"eval_rewards/accuracy_reward": 0.6266666650772095,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23473802208900452,
|
|
"eval_rewards/answer_logprob_reward": 3.021596670150757,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.6094379425048828,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.1580141931772232,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.03777023404836655,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.08813288807868958,
|
|
"eval_rewards/len_reward/p75": 0.226766437292099,
|
|
"eval_rewards/len_reward/var": 0.06537482887506485,
|
|
"eval_rewards/missing_response_penalty": -0.09666667133569717,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0876142680644989,
|
|
"eval_runtime": 173.5451,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8668287396430969,
|
|
"eval_values/max": 1.9913330078125,
|
|
"eval_values/median": -0.82958984375,
|
|
"eval_values/min": -1.998321533203125,
|
|
"eval_values/p25": -0.82958984375,
|
|
"eval_values/p75": -0.82958984375,
|
|
"eval_values/var": 0.12681037187576294,
|
|
"step": 37
|
|
},
|
|
{
|
|
"accuracy_reward": 0.7916666865348816,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 1.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.16579407453536987,
|
|
"adam_stats/lr_effective_max": 1.1624864271198021e-07,
|
|
"adam_stats/lr_effective_mean": -4.2340298476474525e-13,
|
|
"adam_stats/lr_effective_min": -1.1628848994860164e-07,
|
|
"adam_stats/m_t_max": 0.003072062274441123,
|
|
"adam_stats/m_t_mean": -1.9277281160245963e-11,
|
|
"adam_stats/m_t_min": -0.0031752227805554867,
|
|
"adam_stats/v_t_max": 2.3140813937061466e-05,
|
|
"adam_stats/v_t_mean": 1.61747449201749e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.7916666865348816,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 1.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.16579407453536987,
|
|
"all_logprobs": -0.012521984055638313,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -8.25,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.018667710945010185,
|
|
"answer_logprob_reward": 3.0625293254852295,
|
|
"answer_logprob_reward/correct": 3.1005308628082275,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.4849066734313965,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.04804110899567604,
|
|
"answer_logprob_reward/incorrect": 2.918123722076416,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p25": 2.4849066734313965,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.21404796838760376,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.7917594909667969,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.08720266073942184,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 510.26043701171875,
|
|
"completion_length/correct": 448.27630615234375,
|
|
"completion_length/correct/max": 1013.0,
|
|
"completion_length/correct/median": 410.0,
|
|
"completion_length/correct/min": 125.0,
|
|
"completion_length/correct/p25": 301.5,
|
|
"completion_length/correct/p75": 585.25,
|
|
"completion_length/correct/var": 36263.9921875,
|
|
"completion_length/incorrect": 745.7999877929688,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 725.0,
|
|
"completion_length/incorrect/min": 271.0,
|
|
"completion_length/incorrect/p25": 522.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 61569.08203125,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 442.0,
|
|
"completion_length/min": 125.0,
|
|
"completion_length/p25": 326.75,
|
|
"completion_length/p75": 661.5,
|
|
"completion_length/var": 55917.28125,
|
|
"epoch": 0.9620253164556962,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 4006488.0,
|
|
"grad_norm": 0.24734780192375183,
|
|
"kl": 0.013702392578125,
|
|
"learning_rate": 2.012853002380466e-08,
|
|
"len_reward": 0.10989409685134888,
|
|
"len_reward/correct": 0.18586990237236023,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2170129120349884,
|
|
"len_reward/correct/min": -0.40889251232147217,
|
|
"len_reward/correct/p25": 0.05338798090815544,
|
|
"len_reward/correct/p75": 0.3427121341228485,
|
|
"len_reward/correct/var": 0.04710303992033005,
|
|
"len_reward/incorrect": -0.17881394922733307,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.12961409986019135,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.2924572229385376,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03831915184855461,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.1652684509754181,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.05153507739305496,
|
|
"len_reward/p75": 0.30100107192993164,
|
|
"len_reward/var": 0.0671125203371048,
|
|
"loss": -0.7911,
|
|
"mean_logprobs": -0.01220703125,
|
|
"mean_logprobs/var": 0.00010061264038085938,
|
|
"missing_response_penalty": -0.0572916679084301,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.2750000059604645,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2044871747493744,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.05429210141301155,
|
|
"non_smoothed_logprobs/min": -17.0625,
|
|
"num_completions/total": 7296,
|
|
"policy_loss": -0.7911183834075928,
|
|
"policy_loss/max": 0.000813791120890528,
|
|
"policy_loss/median": -0.9994096755981445,
|
|
"policy_loss/min": -0.9997642040252686,
|
|
"policy_loss/p25": -0.9995551109313965,
|
|
"policy_loss/p75": -0.9991562962532043,
|
|
"policy_loss/var": 0.16578398644924164,
|
|
"pred_value_log_probs": -0.6019009351730347,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.671875,
|
|
"pred_value_log_probs/min": -1.1875,
|
|
"pred_value_log_probs/p25": -0.671875,
|
|
"pred_value_log_probs/p75": -0.671875,
|
|
"pred_value_log_probs/var": 0.037637632340192795,
|
|
"reward": 0.7916666865348816,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 1.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.16579407453536987,
|
|
"rewards/accuracy_reward": 0.7916666865348816,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 1.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.16579407453536987,
|
|
"rewards/answer_logprob_reward": 3.0625293254852295,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.7917594909667969,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.08720266073942184,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.10989409685134888,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.1652684509754181,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.05153507739305496,
|
|
"rewards/len_reward/p75": 0.30100107192993164,
|
|
"rewards/len_reward/var": 0.0671125203371048,
|
|
"rewards/missing_response_penalty": -0.0572916679084301,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.05429210141301155,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 38,
|
|
"value_loss": 3.0,
|
|
"values": -0.8338667750358582,
|
|
"values/max": 1.6683349609375,
|
|
"values/median": -0.82958984375,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.82958984375,
|
|
"values/p75": -0.82958984375,
|
|
"values/var": 0.21893833577632904
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": 1.1624864271198021e-07,
|
|
"adam_stats/lr_effective_mean": -4.2340298476474525e-13,
|
|
"adam_stats/lr_effective_min": -1.1628848994860164e-07,
|
|
"adam_stats/m_t_max": 0.003072062274441123,
|
|
"adam_stats/m_t_mean": -1.9277281160245963e-11,
|
|
"adam_stats/m_t_min": -0.0031752227805554867,
|
|
"adam_stats/v_t_max": 2.3140813937061466e-05,
|
|
"adam_stats/v_t_mean": 1.61747449201749e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.9620253164556962,
|
|
"eval_accuracy_reward": 0.6033333539962769,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.24012264609336853,
|
|
"eval_advantages": 0.6033333539962769,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.24012264609336853,
|
|
"eval_all_logprobs": -0.012280795723199844,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -4.625,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.01731371320784092,
|
|
"eval_answer_logprob_reward": 3.0406923294067383,
|
|
"eval_answer_logprob_reward/correct": 3.1116490364074707,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.525728702545166,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.06317559629678726,
|
|
"eval_answer_logprob_reward/incorrect": 2.932767152786255,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.2800227105617523,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.15622638165950775,
|
|
"eval_completion_length": 512.9666748046875,
|
|
"eval_completion_length/correct": 441.5027770996094,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 394.0,
|
|
"eval_completion_length/correct/min": 123.0,
|
|
"eval_completion_length/correct/p25": 289.0,
|
|
"eval_completion_length/correct/p75": 560.0,
|
|
"eval_completion_length/correct/var": 40204.70703125,
|
|
"eval_completion_length/incorrect": 621.6638793945312,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 556.0,
|
|
"eval_completion_length/incorrect/min": 155.0,
|
|
"eval_completion_length/incorrect/p25": 384.0,
|
|
"eval_completion_length/incorrect/p75": 923.0,
|
|
"eval_completion_length/incorrect/var": 79311.71875,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 456.0,
|
|
"eval_completion_length/min": 123.0,
|
|
"eval_completion_length/p25": 313.0,
|
|
"eval_completion_length/p75": 660.25,
|
|
"eval_completion_length/var": 63297.6796875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 6302489.0,
|
|
"eval_len_reward": 0.0595286525785923,
|
|
"eval_len_reward/correct": 0.17562632262706757,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.18323656916618347,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": 0.05504395067691803,
|
|
"eval_len_reward/correct/p75": 0.32739973068237305,
|
|
"eval_len_reward/correct/var": 0.04462508112192154,
|
|
"eval_len_reward/incorrect": -0.11705687642097473,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": 0.0,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.20975428819656372,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.02766287699341774,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.03335507586598396,
|
|
"eval_len_reward/p75": 0.2350156605243683,
|
|
"eval_len_reward/var": 0.05835144594311714,
|
|
"eval_mean_logprobs": -0.0123291015625,
|
|
"eval_mean_logprobs/var": 7.43865966796875e-05,
|
|
"eval_missing_response_penalty": -0.09000000357627869,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.22689077258110046,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.17689786851406097,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.0821739062666893,
|
|
"eval_num_completions/total": 11400,
|
|
"eval_pred_value_log_probs": -0.6375287771224976,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.6875,
|
|
"eval_pred_value_log_probs/min": -1.0234375,
|
|
"eval_pred_value_log_probs/p25": -0.6875,
|
|
"eval_pred_value_log_probs/p75": -0.6875,
|
|
"eval_pred_value_log_probs/var": 0.02683435007929802,
|
|
"eval_reward": 0.6033333539962769,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.24012264609336853,
|
|
"eval_rewards/accuracy_reward": 0.6033333539962769,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.24012264609336853,
|
|
"eval_rewards/answer_logprob_reward": 3.0406923294067383,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.4271163940429688,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.15622638165950775,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.0595286525785923,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.03335507586598396,
|
|
"eval_rewards/len_reward/p75": 0.2350156605243683,
|
|
"eval_rewards/len_reward/var": 0.05835144594311714,
|
|
"eval_rewards/missing_response_penalty": -0.09000000357627869,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.0821739062666893,
|
|
"eval_runtime": 177.0796,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8462945818901062,
|
|
"eval_values/max": 1.9913215637207031,
|
|
"eval_values/median": -0.8076171875,
|
|
"eval_values/min": -1.9895877838134766,
|
|
"eval_values/p25": -0.8076171875,
|
|
"eval_values/p75": -0.8076171875,
|
|
"eval_values/var": 0.1265174001455307,
|
|
"step": 38
|
|
},
|
|
{
|
|
"accuracy_reward": 0.71875,
|
|
"accuracy_reward/correct": 1.0,
|
|
"accuracy_reward/correct/max": 1.0,
|
|
"accuracy_reward/correct/median": 1.0,
|
|
"accuracy_reward/correct/min": 1.0,
|
|
"accuracy_reward/correct/p25": 1.0,
|
|
"accuracy_reward/correct/p75": 1.0,
|
|
"accuracy_reward/correct/var": 0.0,
|
|
"accuracy_reward/incorrect": 0.0,
|
|
"accuracy_reward/incorrect/max": 0.0,
|
|
"accuracy_reward/incorrect/median": 0.0,
|
|
"accuracy_reward/incorrect/min": 0.0,
|
|
"accuracy_reward/incorrect/p25": 0.0,
|
|
"accuracy_reward/incorrect/p75": 0.0,
|
|
"accuracy_reward/incorrect/var": 0.0,
|
|
"accuracy_reward/max": 1.0,
|
|
"accuracy_reward/median": 1.0,
|
|
"accuracy_reward/min": 0.0,
|
|
"accuracy_reward/p25": 0.0,
|
|
"accuracy_reward/p75": 1.0,
|
|
"accuracy_reward/var": 0.20320680737495422,
|
|
"adam_stats/lr_effective_max": -0.0,
|
|
"adam_stats/lr_effective_mean": 0.0,
|
|
"adam_stats/lr_effective_min": -0.0,
|
|
"adam_stats/m_t_max": 0.003430139273405075,
|
|
"adam_stats/m_t_mean": -2.669468639193706e-11,
|
|
"adam_stats/m_t_min": -0.0036267435643821955,
|
|
"adam_stats/v_t_max": 2.3161932404036634e-05,
|
|
"adam_stats/v_t_mean": 1.6231577713854417e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"advantages": 0.71875,
|
|
"advantages/max": 1.0,
|
|
"advantages/median": 1.0,
|
|
"advantages/min": 0.0,
|
|
"advantages/p25": 0.0,
|
|
"advantages/p75": 1.0,
|
|
"advantages/var": 0.20320680737495422,
|
|
"all_logprobs": -0.011965964920818806,
|
|
"all_logprobs/max": 0.0,
|
|
"all_logprobs/median": 0.0,
|
|
"all_logprobs/min": -5.28125,
|
|
"all_logprobs/p25": 0.0,
|
|
"all_logprobs/p75": 0.0,
|
|
"all_logprobs/var": 0.01780596747994423,
|
|
"answer_logprob_reward": 2.9355921745300293,
|
|
"answer_logprob_reward/correct": 3.0717215538024902,
|
|
"answer_logprob_reward/correct/max": 3.178053855895996,
|
|
"answer_logprob_reward/correct/median": 3.178053855895996,
|
|
"answer_logprob_reward/correct/min": 2.079441547393799,
|
|
"answer_logprob_reward/correct/p25": 3.178053855895996,
|
|
"answer_logprob_reward/correct/p75": 3.178053855895996,
|
|
"answer_logprob_reward/correct/var": 0.0693558007478714,
|
|
"answer_logprob_reward/incorrect": 2.5877063274383545,
|
|
"answer_logprob_reward/incorrect/max": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/median": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/min": 1.2321436405181885,
|
|
"answer_logprob_reward/incorrect/p25": 1.7917594909667969,
|
|
"answer_logprob_reward/incorrect/p75": 3.178053855895996,
|
|
"answer_logprob_reward/incorrect/var": 0.5624627470970154,
|
|
"answer_logprob_reward/max": 3.178053855895996,
|
|
"answer_logprob_reward/median": 3.178053855895996,
|
|
"answer_logprob_reward/min": 1.2321436405181885,
|
|
"answer_logprob_reward/p25": 3.178053855895996,
|
|
"answer_logprob_reward/p75": 3.178053855895996,
|
|
"answer_logprob_reward/var": 0.25342875719070435,
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 532.90625,
|
|
"completion_length/correct": 436.1449279785156,
|
|
"completion_length/correct/max": 972.0,
|
|
"completion_length/correct/median": 403.0,
|
|
"completion_length/correct/min": 120.0,
|
|
"completion_length/correct/p25": 312.5,
|
|
"completion_length/correct/p75": 533.5,
|
|
"completion_length/correct/var": 33634.80078125,
|
|
"completion_length/incorrect": 780.1851806640625,
|
|
"completion_length/incorrect/max": 1024.0,
|
|
"completion_length/incorrect/median": 880.0,
|
|
"completion_length/incorrect/min": 165.0,
|
|
"completion_length/incorrect/p25": 554.0,
|
|
"completion_length/incorrect/p75": 1024.0,
|
|
"completion_length/incorrect/var": 75946.609375,
|
|
"completion_length/max": 1024.0,
|
|
"completion_length/median": 447.0,
|
|
"completion_length/min": 120.0,
|
|
"completion_length/p25": 337.5,
|
|
"completion_length/p75": 683.5,
|
|
"completion_length/var": 69251.984375,
|
|
"epoch": 0.9873417721518988,
|
|
"format_reward": 0.0,
|
|
"format_reward/correct": 0.0,
|
|
"format_reward/correct/max": 0.0,
|
|
"format_reward/correct/median": 0.0,
|
|
"format_reward/correct/min": 0.0,
|
|
"format_reward/correct/p25": 0.0,
|
|
"format_reward/correct/p75": 0.0,
|
|
"format_reward/correct/var": 0.0,
|
|
"format_reward/incorrect": 0.0,
|
|
"format_reward/incorrect/max": 0.0,
|
|
"format_reward/incorrect/median": 0.0,
|
|
"format_reward/incorrect/min": 0.0,
|
|
"format_reward/incorrect/p25": 0.0,
|
|
"format_reward/incorrect/p75": 0.0,
|
|
"format_reward/incorrect/var": 0.0,
|
|
"format_reward/max": 0.0,
|
|
"format_reward/median": 0.0,
|
|
"format_reward/min": 0.0,
|
|
"format_reward/p25": 0.0,
|
|
"format_reward/p75": 0.0,
|
|
"format_reward/var": 0.0,
|
|
"generated_tokens/total": 4108806.0,
|
|
"grad_norm": 0.2184079885482788,
|
|
"kl": 0.013671875,
|
|
"learning_rate": 0.0,
|
|
"len_reward": 0.09715957939624786,
|
|
"len_reward/correct": 0.2102111130952835,
|
|
"len_reward/correct/max": 0.5,
|
|
"len_reward/correct/median": 0.2156357318162918,
|
|
"len_reward/correct/min": -0.4198760390281677,
|
|
"len_reward/correct/p25": 0.10082200914621353,
|
|
"len_reward/correct/p75": 0.3562450110912323,
|
|
"len_reward/correct/var": 0.03685981035232544,
|
|
"len_reward/incorrect": -0.1917499303817749,
|
|
"len_reward/incorrect/max": 0.0,
|
|
"len_reward/incorrect/median": -0.1574459969997406,
|
|
"len_reward/incorrect/min": -0.5,
|
|
"len_reward/incorrect/p25": -0.3096248507499695,
|
|
"len_reward/incorrect/p75": 0.0,
|
|
"len_reward/incorrect/var": 0.03601037338376045,
|
|
"len_reward/max": 0.5,
|
|
"len_reward/median": 0.11759082227945328,
|
|
"len_reward/min": -0.5,
|
|
"len_reward/p25": -0.03279658034443855,
|
|
"len_reward/p75": 0.2950369417667389,
|
|
"len_reward/var": 0.06926378607749939,
|
|
"loss": -0.7182,
|
|
"mean_logprobs": -0.0118408203125,
|
|
"mean_logprobs/var": 6.771087646484375e-05,
|
|
"missing_response_penalty": -0.125,
|
|
"missing_response_penalty/correct": 0.0,
|
|
"missing_response_penalty/correct/max": 0.0,
|
|
"missing_response_penalty/correct/median": 0.0,
|
|
"missing_response_penalty/correct/min": 0.0,
|
|
"missing_response_penalty/correct/p25": 0.0,
|
|
"missing_response_penalty/correct/p75": 0.0,
|
|
"missing_response_penalty/correct/var": 0.0,
|
|
"missing_response_penalty/incorrect": -0.4444444477558136,
|
|
"missing_response_penalty/incorrect/max": 0.0,
|
|
"missing_response_penalty/incorrect/median": 0.0,
|
|
"missing_response_penalty/incorrect/min": -1.0,
|
|
"missing_response_penalty/incorrect/p25": -1.0,
|
|
"missing_response_penalty/incorrect/p75": 0.0,
|
|
"missing_response_penalty/incorrect/var": 0.2515723407268524,
|
|
"missing_response_penalty/max": 0.0,
|
|
"missing_response_penalty/median": 0.0,
|
|
"missing_response_penalty/min": -1.0,
|
|
"missing_response_penalty/p25": 0.0,
|
|
"missing_response_penalty/p75": 0.0,
|
|
"missing_response_penalty/var": 0.10994765162467957,
|
|
"non_smoothed_logprobs/min": -17.625,
|
|
"num_completions/total": 7488,
|
|
"policy_loss": -0.6661717891693115,
|
|
"policy_loss/max": 0.000675500079523772,
|
|
"policy_loss/median": -0.9994784593582153,
|
|
"policy_loss/min": -0.9996159672737122,
|
|
"policy_loss/p25": -0.9995781183242798,
|
|
"policy_loss/p75": 0.0003850152716040611,
|
|
"policy_loss/var": 0.23185908794403076,
|
|
"pred_value_log_probs": -0.644974410533905,
|
|
"pred_value_log_probs/max": 0.0,
|
|
"pred_value_log_probs/median": -0.6875,
|
|
"pred_value_log_probs/min": -1.21875,
|
|
"pred_value_log_probs/p25": -0.6875,
|
|
"pred_value_log_probs/p75": -0.6875,
|
|
"pred_value_log_probs/var": 0.04008783400058746,
|
|
"reward": 0.71875,
|
|
"reward/max": 1.0,
|
|
"reward/median": 1.0,
|
|
"reward/min": 0.0,
|
|
"reward/p25": 0.0,
|
|
"reward/p75": 1.0,
|
|
"reward/var": 0.20320680737495422,
|
|
"rewards/accuracy_reward": 0.71875,
|
|
"rewards/accuracy_reward/max": 1.0,
|
|
"rewards/accuracy_reward/median": 1.0,
|
|
"rewards/accuracy_reward/min": 0.0,
|
|
"rewards/accuracy_reward/p25": 0.0,
|
|
"rewards/accuracy_reward/p75": 1.0,
|
|
"rewards/accuracy_reward/var": 0.20320680737495422,
|
|
"rewards/answer_logprob_reward": 2.9355921745300293,
|
|
"rewards/answer_logprob_reward/max": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/median": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/min": 1.2321436405181885,
|
|
"rewards/answer_logprob_reward/p25": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/p75": 3.178053855895996,
|
|
"rewards/answer_logprob_reward/var": 0.25342875719070435,
|
|
"rewards/format_reward": 0.0,
|
|
"rewards/format_reward/max": 0.0,
|
|
"rewards/format_reward/median": 0.0,
|
|
"rewards/format_reward/min": 0.0,
|
|
"rewards/format_reward/p25": 0.0,
|
|
"rewards/format_reward/p75": 0.0,
|
|
"rewards/format_reward/var": 0.0,
|
|
"rewards/len_reward": 0.09715957939624786,
|
|
"rewards/len_reward/max": 0.5,
|
|
"rewards/len_reward/median": 0.11759082227945328,
|
|
"rewards/len_reward/min": -0.5,
|
|
"rewards/len_reward/p25": -0.03279658034443855,
|
|
"rewards/len_reward/p75": 0.2950369417667389,
|
|
"rewards/len_reward/var": 0.06926378607749939,
|
|
"rewards/missing_response_penalty": -0.125,
|
|
"rewards/missing_response_penalty/max": 0.0,
|
|
"rewards/missing_response_penalty/median": 0.0,
|
|
"rewards/missing_response_penalty/min": -1.0,
|
|
"rewards/missing_response_penalty/p25": 0.0,
|
|
"rewards/missing_response_penalty/p75": 0.0,
|
|
"rewards/missing_response_penalty/var": 0.10994765162467957,
|
|
"smoothed_logprobs/min": -1.0,
|
|
"step": 39,
|
|
"value_loss": 2.859375,
|
|
"values": -0.7773435115814209,
|
|
"values/max": 1.5750732421875,
|
|
"values/median": -0.8076171875,
|
|
"values/min": -2.0,
|
|
"values/p25": -0.8076171875,
|
|
"values/p75": -0.8076171875,
|
|
"values/var": 0.21612520515918732
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": -0.0,
|
|
"adam_stats/lr_effective_mean": 0.0,
|
|
"adam_stats/lr_effective_min": -0.0,
|
|
"adam_stats/m_t_max": 0.003430139273405075,
|
|
"adam_stats/m_t_mean": -2.669468639193706e-11,
|
|
"adam_stats/m_t_min": -0.0036267435643821955,
|
|
"adam_stats/v_t_max": 2.3161932404036634e-05,
|
|
"adam_stats/v_t_mean": 1.6231577713854417e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.9873417721518988,
|
|
"eval_accuracy_reward": 0.6399999856948853,
|
|
"eval_accuracy_reward/correct": 1.0,
|
|
"eval_accuracy_reward/correct/max": 1.0,
|
|
"eval_accuracy_reward/correct/median": 1.0,
|
|
"eval_accuracy_reward/correct/min": 1.0,
|
|
"eval_accuracy_reward/correct/p25": 1.0,
|
|
"eval_accuracy_reward/correct/p75": 1.0,
|
|
"eval_accuracy_reward/correct/var": 0.0,
|
|
"eval_accuracy_reward/incorrect": 0.0,
|
|
"eval_accuracy_reward/incorrect/max": 0.0,
|
|
"eval_accuracy_reward/incorrect/median": 0.0,
|
|
"eval_accuracy_reward/incorrect/min": 0.0,
|
|
"eval_accuracy_reward/incorrect/p25": 0.0,
|
|
"eval_accuracy_reward/incorrect/p75": 0.0,
|
|
"eval_accuracy_reward/incorrect/var": 0.0,
|
|
"eval_accuracy_reward/max": 1.0,
|
|
"eval_accuracy_reward/median": 1.0,
|
|
"eval_accuracy_reward/min": 0.0,
|
|
"eval_accuracy_reward/p25": 0.0,
|
|
"eval_accuracy_reward/p75": 1.0,
|
|
"eval_accuracy_reward/var": 0.23117057979106903,
|
|
"eval_advantages": 0.6399999856948853,
|
|
"eval_advantages/max": 1.0,
|
|
"eval_advantages/median": 1.0,
|
|
"eval_advantages/min": 0.0,
|
|
"eval_advantages/p25": 0.0,
|
|
"eval_advantages/p75": 1.0,
|
|
"eval_advantages/var": 0.23117057979106903,
|
|
"eval_all_logprobs": -0.012546074576675892,
|
|
"eval_all_logprobs/max": 0.0,
|
|
"eval_all_logprobs/median": 0.0,
|
|
"eval_all_logprobs/min": -5.84375,
|
|
"eval_all_logprobs/p25": 0.0,
|
|
"eval_all_logprobs/p75": 0.0,
|
|
"eval_all_logprobs/var": 0.018090752884745598,
|
|
"eval_answer_logprob_reward": 3.054785966873169,
|
|
"eval_answer_logprob_reward/correct": 3.12078857421875,
|
|
"eval_answer_logprob_reward/correct/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/min": 2.1202635765075684,
|
|
"eval_answer_logprob_reward/correct/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/correct/var": 0.0633377954363823,
|
|
"eval_answer_logprob_reward/incorrect": 2.9374477863311768,
|
|
"eval_answer_logprob_reward/incorrect/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/incorrect/p25": 2.525728702545166,
|
|
"eval_answer_logprob_reward/incorrect/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/incorrect/var": 0.23042640089988708,
|
|
"eval_answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_answer_logprob_reward/var": 0.13069075345993042,
|
|
"eval_completion_length": 522.9367065429688,
|
|
"eval_completion_length/correct": 466.28125,
|
|
"eval_completion_length/correct/max": 1024.0,
|
|
"eval_completion_length/correct/median": 427.0,
|
|
"eval_completion_length/correct/min": 123.0,
|
|
"eval_completion_length/correct/p25": 282.5,
|
|
"eval_completion_length/correct/p75": 624.0,
|
|
"eval_completion_length/correct/var": 45490.38671875,
|
|
"eval_completion_length/incorrect": 623.6574096679688,
|
|
"eval_completion_length/incorrect/max": 1024.0,
|
|
"eval_completion_length/incorrect/median": 563.0,
|
|
"eval_completion_length/incorrect/min": 161.0,
|
|
"eval_completion_length/incorrect/p25": 354.25,
|
|
"eval_completion_length/incorrect/p75": 982.0,
|
|
"eval_completion_length/incorrect/var": 91662.375,
|
|
"eval_completion_length/max": 1024.0,
|
|
"eval_completion_length/median": 462.0,
|
|
"eval_completion_length/min": 123.0,
|
|
"eval_completion_length/p25": 304.75,
|
|
"eval_completion_length/p75": 716.75,
|
|
"eval_completion_length/var": 67586.8046875,
|
|
"eval_format_reward": 0.0,
|
|
"eval_format_reward/correct": 0.0,
|
|
"eval_format_reward/correct/max": 0.0,
|
|
"eval_format_reward/correct/median": 0.0,
|
|
"eval_format_reward/correct/min": 0.0,
|
|
"eval_format_reward/correct/p25": 0.0,
|
|
"eval_format_reward/correct/p75": 0.0,
|
|
"eval_format_reward/correct/var": 0.0,
|
|
"eval_format_reward/incorrect": 0.0,
|
|
"eval_format_reward/incorrect/max": 0.0,
|
|
"eval_format_reward/incorrect/median": 0.0,
|
|
"eval_format_reward/incorrect/min": 0.0,
|
|
"eval_format_reward/incorrect/p25": 0.0,
|
|
"eval_format_reward/incorrect/p75": 0.0,
|
|
"eval_format_reward/incorrect/var": 0.0,
|
|
"eval_format_reward/max": 0.0,
|
|
"eval_format_reward/median": 0.0,
|
|
"eval_format_reward/min": 0.0,
|
|
"eval_format_reward/p25": 0.0,
|
|
"eval_format_reward/p75": 0.0,
|
|
"eval_format_reward/var": 0.0,
|
|
"eval_generated_tokens/total": 6459370.0,
|
|
"eval_len_reward": 0.021767830476164818,
|
|
"eval_len_reward/correct": 0.12273482978343964,
|
|
"eval_len_reward/correct/max": 0.5,
|
|
"eval_len_reward/correct/median": 0.13476347923278809,
|
|
"eval_len_reward/correct/min": -0.5,
|
|
"eval_len_reward/correct/p25": -0.04037480056285858,
|
|
"eval_len_reward/correct/p75": 0.302950918674469,
|
|
"eval_len_reward/correct/var": 0.061008352786302567,
|
|
"eval_len_reward/incorrect": -0.15772904455661774,
|
|
"eval_len_reward/incorrect/max": 0.0,
|
|
"eval_len_reward/incorrect/median": 0.0,
|
|
"eval_len_reward/incorrect/min": -0.5,
|
|
"eval_len_reward/incorrect/p25": -0.3225032091140747,
|
|
"eval_len_reward/incorrect/p75": 0.0,
|
|
"eval_len_reward/incorrect/var": 0.03720378503203392,
|
|
"eval_len_reward/max": 0.5,
|
|
"eval_len_reward/median": 0.0,
|
|
"eval_len_reward/min": -0.5,
|
|
"eval_len_reward/p25": -0.13391777873039246,
|
|
"eval_len_reward/p75": 0.21725769340991974,
|
|
"eval_len_reward/var": 0.07046949863433838,
|
|
"eval_mean_logprobs": -0.0128173828125,
|
|
"eval_mean_logprobs/var": 0.00011205673217773438,
|
|
"eval_missing_response_penalty": -0.0833333358168602,
|
|
"eval_missing_response_penalty/correct": 0.0,
|
|
"eval_missing_response_penalty/correct/max": 0.0,
|
|
"eval_missing_response_penalty/correct/median": 0.0,
|
|
"eval_missing_response_penalty/correct/min": 0.0,
|
|
"eval_missing_response_penalty/correct/p25": 0.0,
|
|
"eval_missing_response_penalty/correct/p75": 0.0,
|
|
"eval_missing_response_penalty/correct/var": 0.0,
|
|
"eval_missing_response_penalty/incorrect": -0.23148147761821747,
|
|
"eval_missing_response_penalty/incorrect/max": 0.0,
|
|
"eval_missing_response_penalty/incorrect/median": 0.0,
|
|
"eval_missing_response_penalty/incorrect/min": -1.0,
|
|
"eval_missing_response_penalty/incorrect/p25": 0.0,
|
|
"eval_missing_response_penalty/incorrect/p75": 0.0,
|
|
"eval_missing_response_penalty/incorrect/var": 0.17956037819385529,
|
|
"eval_missing_response_penalty/max": 0.0,
|
|
"eval_missing_response_penalty/median": 0.0,
|
|
"eval_missing_response_penalty/min": -1.0,
|
|
"eval_missing_response_penalty/p25": 0.0,
|
|
"eval_missing_response_penalty/p75": 0.0,
|
|
"eval_missing_response_penalty/var": 0.07664436846971512,
|
|
"eval_num_completions/total": 11700,
|
|
"eval_pred_value_log_probs": -0.635452389717102,
|
|
"eval_pred_value_log_probs/max": 0.0,
|
|
"eval_pred_value_log_probs/median": -0.68359375,
|
|
"eval_pred_value_log_probs/min": -1.078125,
|
|
"eval_pred_value_log_probs/p25": -0.68359375,
|
|
"eval_pred_value_log_probs/p75": -0.68359375,
|
|
"eval_pred_value_log_probs/var": 0.026864545419812202,
|
|
"eval_reward": 0.6399999856948853,
|
|
"eval_reward/max": 1.0,
|
|
"eval_reward/median": 1.0,
|
|
"eval_reward/min": 0.0,
|
|
"eval_reward/p25": 0.0,
|
|
"eval_reward/p75": 1.0,
|
|
"eval_reward/var": 0.23117057979106903,
|
|
"eval_rewards/accuracy_reward": 0.6399999856948853,
|
|
"eval_rewards/accuracy_reward/max": 1.0,
|
|
"eval_rewards/accuracy_reward/median": 1.0,
|
|
"eval_rewards/accuracy_reward/min": 0.0,
|
|
"eval_rewards/accuracy_reward/p25": 0.0,
|
|
"eval_rewards/accuracy_reward/p75": 1.0,
|
|
"eval_rewards/accuracy_reward/var": 0.23117057979106903,
|
|
"eval_rewards/answer_logprob_reward": 3.054785966873169,
|
|
"eval_rewards/answer_logprob_reward/max": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/median": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/min": 1.8325815200805664,
|
|
"eval_rewards/answer_logprob_reward/p25": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/p75": 3.2188758850097656,
|
|
"eval_rewards/answer_logprob_reward/var": 0.13069075345993042,
|
|
"eval_rewards/format_reward": 0.0,
|
|
"eval_rewards/format_reward/max": 0.0,
|
|
"eval_rewards/format_reward/median": 0.0,
|
|
"eval_rewards/format_reward/min": 0.0,
|
|
"eval_rewards/format_reward/p25": 0.0,
|
|
"eval_rewards/format_reward/p75": 0.0,
|
|
"eval_rewards/format_reward/var": 0.0,
|
|
"eval_rewards/len_reward": 0.021767830476164818,
|
|
"eval_rewards/len_reward/max": 0.5,
|
|
"eval_rewards/len_reward/median": 0.0,
|
|
"eval_rewards/len_reward/min": -0.5,
|
|
"eval_rewards/len_reward/p25": -0.13391777873039246,
|
|
"eval_rewards/len_reward/p75": 0.21725769340991974,
|
|
"eval_rewards/len_reward/var": 0.07046949863433838,
|
|
"eval_rewards/missing_response_penalty": -0.0833333358168602,
|
|
"eval_rewards/missing_response_penalty/max": 0.0,
|
|
"eval_rewards/missing_response_penalty/median": 0.0,
|
|
"eval_rewards/missing_response_penalty/min": -1.0,
|
|
"eval_rewards/missing_response_penalty/p25": 0.0,
|
|
"eval_rewards/missing_response_penalty/p75": 0.0,
|
|
"eval_rewards/missing_response_penalty/var": 0.07664436846971512,
|
|
"eval_runtime": 175.7226,
|
|
"eval_samples_per_second": 0.0,
|
|
"eval_steps_per_second": 0.0,
|
|
"eval_values": -0.8476092219352722,
|
|
"eval_values/max": 1.9911956787109375,
|
|
"eval_values/median": -0.80859375,
|
|
"eval_values/min": -1.9981098175048828,
|
|
"eval_values/p25": -0.80859375,
|
|
"eval_values/p75": -0.80859375,
|
|
"eval_values/var": 0.12626203894615173,
|
|
"step": 39
|
|
},
|
|
{
|
|
"adam_stats/lr_effective_max": -0.0,
|
|
"adam_stats/lr_effective_mean": 0.0,
|
|
"adam_stats/lr_effective_min": -0.0,
|
|
"adam_stats/m_t_max": 0.003430139273405075,
|
|
"adam_stats/m_t_mean": -2.669468639193706e-11,
|
|
"adam_stats/m_t_min": -0.0036267435643821955,
|
|
"adam_stats/v_t_max": 2.3161932404036634e-05,
|
|
"adam_stats/v_t_mean": 1.6231577713854417e-12,
|
|
"adam_stats/v_t_min": 0.0,
|
|
"epoch": 0.9873417721518988,
|
|
"step": 39,
|
|
"total_flos": 0.0,
|
|
"train_loss": -0.7422515612382156,
|
|
"train_runtime": 31151.0975,
|
|
"train_samples_per_second": 0.241,
|
|
"train_steps_per_second": 0.001
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 39,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 1,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 24,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|