From 27ea6f94f0b48c3a7188672fae7e60e36e179640 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Wed, 29 Jul 2026 01:20:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-ronpo-full-expect-s42 Source: Original Platform --- .gitattributes | 36 ++++ README.md | 25 +++ all_results.json | 9 + chat_template.jinja | 89 +++++++++ config.json | 71 ++++++++ config.yaml | 54 ++++++ generation_config.json | 12 ++ model.safetensors | 3 + run_status.json | 18 ++ tokenizer.json | 3 + tokenizer_config.json | 30 +++ train_results.json | 9 + trainer_state.json | 403 +++++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 14 files changed, 765 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 run_status.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..3fa581c --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- qwen3 +- preference-optimization +--- + +# qwen3-8b-ronpo-full-expect-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: RONPO-full-expect-best-vs-adversary +- Base model: `Qwen/Qwen3-8B` with non-thinking generation protocol +- Seed: 42 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3` +- Uploaded at UTC: 2026-07-12T01:30:33Z +- Run status metadata: `{"adversary_selection": "sample", "completed_at": "2026-07-11T23:03:13Z", "config": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3/config.yaml", "dataset": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/full_expect_bestadv_sample1", "method": "ronpo-ablation", "mode": "full_expect", "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3", "pair_variant": "bestadv_sample1", "pairs": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/pairs/full_expect_bestadv_sample1", "returncode": 0, "ronpo_policy_pair_mode": "best_vs_adversary", "seed": 42, "source_dataset": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", "status": "completed", "train_metrics_exists": true, "trainer_state_exists": true}` + +Qwen3-8B-scale-up-IFEval-diagnostic-541-prompts + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..16091b3 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.06652991073903643, + "total_flos": 0.0, + "train_loss": 1.094520975748698, + "train_runtime": 689.8738, + "train_samples": 18037, + "train_samples_per_second": 1.739, + "train_steps_per_second": 0.109 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..41594ea --- /dev/null +++ b/config.yaml @@ -0,0 +1,54 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: eager +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/full_expect_bestadv_sample1: 1.0 +dataset_splits: + - train + - test +preprocessing_num_workers: 12 +bf16: true +loss_type: ronpo +max_history_t: 1 +history_weights: [1.0] +ronpo_alpha: 1.0 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0 +preference_sft_weight: 0.005 +beta: 10 +learning_rate: 5.0000000000e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +gradient_checkpointing_kwargs: + use_reentrant: false +num_train_epochs: 1 +max_steps: 75 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: "no" +eval_steps: 500 +logging_steps: 5 +log_level: info +generate_during_eval: false +metric_for_best_model: eval_loss +load_best_model_at_end: false +save_strategy: "steps" +save_steps: 1000 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: + - wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3 +run_name: rev-q3-ronpo-full_expect-bestadv_sample1-s42-ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..f6c3203 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c1dc4f346065ac2f9498fd9ad3bb4103e54adc508de8a0ed8058591a48232bd +size 16381517208 diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..3fc1beb --- /dev/null +++ b/run_status.json @@ -0,0 +1,18 @@ +{ + "method": "ronpo-ablation", + "mode": "full_expect", + "pair_variant": "bestadv_sample1", + "ronpo_policy_pair_mode": "best_vs_adversary", + "adversary_selection": "sample", + "seed": 42, + "status": "completed", + "returncode": 0, + "completed_at": "2026-07-11T23:03:13Z", + "source_dataset": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "dataset": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/full_expect_bestadv_sample1", + "pairs": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/pairs/full_expect_bestadv_sample1", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3", + "config": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3/config.yaml", + "train_metrics_exists": true, + "trainer_state_exists": true +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..16091b3 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.06652991073903643, + "total_flos": 0.0, + "train_loss": 1.094520975748698, + "train_runtime": 689.8738, + "train_samples": 18037, + "train_samples_per_second": 1.739, + "train_steps_per_second": 0.109 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..1bcd14a --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,403 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.06652991073903643, + "eval_steps": 500, + "global_step": 75, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004435327382602428, + "grad_norm": 14.0, + "learning_rate": 2.5e-07, + "logits/chosen": -2.144705295562744, + "logits/rejected": -2.0726897716522217, + "logps/chosen": -0.2952641248703003, + "logps/rejected": -0.3042605519294739, + "loss": 0.9013038873672485, + "loss/core": 0.8998275995254517, + "loss/preference_sft": 0.2952641248703003, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.2125720977783203, + "rewards/margins": 1.2788416147232056, + "rewards/rejected": 0.9337307810783386, + "ronpo/logit": 0.1278841495513916, + "ronpo/residual": -0.8721157908439636, + "ronpo/residual_abs": 0.9368273615837097, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 5 + }, + { + "epoch": 0.008870654765204856, + "grad_norm": 210.0, + "learning_rate": 4.997252228714278e-07, + "logits/chosen": -2.165788412094116, + "logits/rejected": -2.1329445838928223, + "logps/chosen": -0.24870172142982483, + "logps/rejected": -0.28110265731811523, + "loss": 0.9554009437561035, + "loss/core": 0.9541573524475098, + "loss/preference_sft": 0.24870172142982483, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.0928869247436523, + "rewards/margins": 1.3761483430862427, + "rewards/rejected": 1.7167381048202515, + "ronpo/logit": 0.13761483132839203, + "ronpo/residual": -0.862385094165802, + "ronpo/residual_abs": 0.9450987577438354, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 10 + }, + { + "epoch": 0.013305982147807285, + "grad_norm": 11.3125, + "learning_rate": 4.9017129689421e-07, + "logits/chosen": -2.123521089553833, + "logits/rejected": -2.0528640747070312, + "logps/chosen": -0.30137166380882263, + "logps/rejected": -0.2893705666065216, + "loss": 0.9385660290718079, + "loss/core": 0.937059223651886, + "loss/preference_sft": 0.30137166380882263, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6111516952514648, + "rewards/margins": 0.4742206931114197, + "rewards/rejected": 1.13693106174469, + "ronpo/logit": 0.04742206260561943, + "ronpo/residual": -0.9525778889656067, + "ronpo/residual_abs": 0.9525778889656067, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 15 + }, + { + "epoch": 0.017741309530409713, + "grad_norm": 12.5, + "learning_rate": 4.6747659310219757e-07, + "logits/chosen": -2.2110671997070312, + "logits/rejected": -2.0166139602661133, + "logps/chosen": -0.2711886465549469, + "logps/rejected": -0.29270288348197937, + "loss": 1.0191020965576172, + "loss/core": 1.0177462100982666, + "loss/preference_sft": 0.2711886465549469, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.056633949279785, + "rewards/margins": 0.49368467926979065, + "rewards/rejected": 1.5629494190216064, + "ronpo/logit": 0.04936846345663071, + "ronpo/residual": -0.9506314992904663, + "ronpo/residual_abs": 0.9506314992904663, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 20 + }, + { + "epoch": 0.02217663691301214, + "grad_norm": 6.59375, + "learning_rate": 4.3288283381591725e-07, + "logits/chosen": -2.171949625015259, + "logits/rejected": -2.0527703762054443, + "logps/chosen": -0.28833886981010437, + "logps/rejected": -0.29443103075027466, + "loss": 0.9356898069381714, + "loss/core": 0.934248149394989, + "loss/preference_sft": 0.28833886981010437, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5931298732757568, + "rewards/margins": 0.5489386320114136, + "rewards/rejected": 1.0441911220550537, + "ronpo/logit": 0.0548938624560833, + "ronpo/residual": -0.945106029510498, + "ronpo/residual_abs": 0.945106029510498, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 25 + }, + { + "epoch": 0.02661196429561457, + "grad_norm": 49.75, + "learning_rate": 3.882827885312998e-07, + "logits/chosen": -2.277024269104004, + "logits/rejected": -2.2178688049316406, + "logps/chosen": -0.30705174803733826, + "logps/rejected": -0.3033962845802307, + "loss": 1.0691256523132324, + "loss/core": 1.0675902366638184, + "loss/preference_sft": 0.30705174803733826, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.114480495452881, + "rewards/margins": 1.2642418146133423, + "rewards/rejected": 0.8502384424209595, + "ronpo/logit": 0.126424178481102, + "ronpo/residual": -0.8735758662223816, + "ronpo/residual_abs": 0.9844897389411926, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 30 + }, + { + "epoch": 0.031047291678216998, + "grad_norm": 19.75, + "learning_rate": 3.361167125710832e-07, + "logits/chosen": -2.1667983531951904, + "logits/rejected": -2.209552764892578, + "logps/chosen": -0.29950323700904846, + "logps/rejected": -0.3074646592140198, + "loss": 0.9153361320495605, + "loss/core": 0.9138385057449341, + "loss/preference_sft": 0.29950323700904846, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.2793786525726318, + "rewards/margins": 0.6898711919784546, + "rewards/rejected": 0.5895074009895325, + "ronpo/logit": 0.06898711621761322, + "ronpo/residual": -0.9310129284858704, + "ronpo/residual_abs": 0.9501181840896606, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 35 + }, + { + "epoch": 0.035482619060819426, + "grad_norm": 14.5, + "learning_rate": 2.7923883049302066e-07, + "logits/chosen": -2.0911293029785156, + "logits/rejected": -1.996106743812561, + "logps/chosen": -0.2962394654750824, + "logps/rejected": -0.2883327603340149, + "loss": 1.2223423719406128, + "loss/core": 1.2208611965179443, + "loss/preference_sft": 0.2962394654750824, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.883702516555786, + "rewards/margins": 1.178934097290039, + "rewards/rejected": 1.7047685384750366, + "ronpo/logit": 0.11789341270923615, + "ronpo/residual": -0.882106602191925, + "ronpo/residual_abs": 0.9954751133918762, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 40 + }, + { + "epoch": 0.03991794644342186, + "grad_norm": 17.5, + "learning_rate": 2.2076116950697937e-07, + "logits/chosen": -2.2753586769104004, + "logits/rejected": -2.194331645965576, + "logps/chosen": -0.2942467927932739, + "logps/rejected": -0.2941977381706238, + "loss": 0.960641086101532, + "loss/core": 0.959169864654541, + "loss/preference_sft": 0.2942467927932739, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3440639972686768, + "rewards/margins": 1.245705008506775, + "rewards/rejected": 1.0983591079711914, + "ronpo/logit": 0.12457048892974854, + "ronpo/residual": -0.8754295110702515, + "ronpo/residual_abs": 0.9467538595199585, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 45 + }, + { + "epoch": 0.04435327382602428, + "grad_norm": 6.03125, + "learning_rate": 1.6388328742891678e-07, + "logits/chosen": -2.3468501567840576, + "logits/rejected": -2.2160422801971436, + "logps/chosen": -0.2907429337501526, + "logps/rejected": -0.299108624458313, + "loss": 1.6810009479522705, + "loss/core": 1.6795470714569092, + "loss/preference_sft": 0.2907429337501526, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.491801381111145, + "rewards/margins": -0.29926931858062744, + "rewards/rejected": 1.7910706996917725, + "ronpo/logit": -0.029926935210824013, + "ronpo/residual": -1.0299268960952759, + "ronpo/residual_abs": 1.0562489032745361, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 50 + }, + { + "epoch": 0.048788601208626714, + "grad_norm": 5.1875, + "learning_rate": 1.1171721146870014e-07, + "logits/chosen": -2.2220540046691895, + "logits/rejected": -2.159651279449463, + "logps/chosen": -0.29590311646461487, + "logps/rejected": -0.3085595369338989, + "loss": 1.2529202699661255, + "loss/core": 1.2514407634735107, + "loss/preference_sft": 0.29590311646461487, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7223021984100342, + "rewards/margins": 0.2412434071302414, + "rewards/rejected": 1.4810588359832764, + "ronpo/logit": 0.02412433736026287, + "ronpo/residual": -0.9758756756782532, + "ronpo/residual_abs": 1.0081208944320679, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 55 + }, + { + "epoch": 0.05322392859122914, + "grad_norm": 121.0, + "learning_rate": 6.711716618408281e-08, + "logits/chosen": -1.9089536666870117, + "logits/rejected": -1.977799654006958, + "logps/chosen": -0.3068787455558777, + "logps/rejected": -0.29860004782676697, + "loss": 1.5921037197113037, + "loss/core": 1.590569257736206, + "loss/preference_sft": 0.3068787455558777, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.075084924697876, + "rewards/margins": 1.1153666973114014, + "rewards/rejected": 1.9597183465957642, + "ronpo/logit": 0.1115366593003273, + "ronpo/residual": -0.8884633183479309, + "ronpo/residual_abs": 0.9980802536010742, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 60 + }, + { + "epoch": 0.05765925597383157, + "grad_norm": 25.25, + "learning_rate": 3.2523406897802444e-08, + "logits/chosen": -2.136443614959717, + "logits/rejected": -2.083848476409912, + "logps/chosen": -0.2734604477882385, + "logps/rejected": -0.28085511922836304, + "loss": 0.8904159665107727, + "loss/core": 0.88904869556427, + "loss/preference_sft": 0.2734604477882385, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9123761653900146, + "rewards/margins": 0.9041536450386047, + "rewards/rejected": 1.0082225799560547, + "ronpo/logit": 0.09041537344455719, + "ronpo/residual": -0.909584641456604, + "ronpo/residual_abs": 0.93060702085495, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 65 + }, + { + "epoch": 0.062094583356433995, + "grad_norm": 8.1875, + "learning_rate": 9.828703105789981e-09, + "logits/chosen": -2.1384823322296143, + "logits/rejected": -2.0494384765625, + "logps/chosen": -0.29097050428390503, + "logps/rejected": -0.3023843765258789, + "loss": 0.8998498916625977, + "loss/core": 0.8983950614929199, + "loss/preference_sft": 0.29097050428390503, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.2245758771896362, + "rewards/margins": 0.5744353532791138, + "rewards/rejected": 0.6501404643058777, + "ronpo/logit": 0.057443536818027496, + "ronpo/residual": -0.9425565004348755, + "ronpo/residual_abs": 0.9425565004348755, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 70 + }, + { + "epoch": 0.06652991073903643, + "grad_norm": 59.25, + "learning_rate": 2.7477712857215675e-10, + "logits/chosen": -2.183204412460327, + "logits/rejected": -2.0384721755981445, + "logps/chosen": -0.27401086688041687, + "logps/rejected": -0.27478593587875366, + "loss": 1.1840168237686157, + "loss/core": 1.182646632194519, + "loss/preference_sft": 0.27401086688041687, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6213176250457764, + "rewards/margins": 0.029666978865861893, + "rewards/rejected": 1.5916506052017212, + "ronpo/logit": 0.002966696862131357, + "ronpo/residual": -0.9970332980155945, + "ronpo/residual_abs": 0.9970332980155945, + "ronpo/target": 1.0, + "ronpo/target_abs": 1.0, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 75 + }, + { + "epoch": 0.06652991073903643, + "step": 75, + "total_flos": 0.0, + "train_loss": 1.094520975748698, + "train_runtime": 689.8738, + "train_samples_per_second": 1.739, + "train_steps_per_second": 0.109 + } + ], + "logging_steps": 5, + "max_steps": 75, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..c4cf323 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b68d7e7de68f2b56eab105f77308db7837d47e25f98b0ef6a9e7b3a039a213da +size 7185