promotion/qwen3-8b-ronpo-full-expect-s42
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: promotion/qwen3-8b-ronpo-full-expect-s42
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-29 01:20:10 +08:00
commit 27ea6f94f0
14 changed files with 765 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

25
README.md Normal file
View File

@@ -0,0 +1,25 @@
---
library_name: transformers
base_model: Qwen/Qwen3-8B
tags:
- ronpo
- mnpo
- qwen3
- preference-optimization
---
# qwen3-8b-ronpo-full-expect-s42
Research checkpoint for the RONPO AAAI revision experiments.
- Method: RONPO-full-expect-best-vs-adversary
- Base model: `Qwen/Qwen3-8B` with non-thinking generation protocol
- Seed: 42
- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3`
- Uploaded at UTC: 2026-07-12T01:30:33Z
- Run status metadata: `{"adversary_selection": "sample", "completed_at": "2026-07-11T23:03:13Z", "config": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3/config.yaml", "dataset": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/full_expect_bestadv_sample1", "method": "ronpo-ablation", "mode": "full_expect", "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3", "pair_variant": "bestadv_sample1", "pairs": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/pairs/full_expect_bestadv_sample1", "returncode": 0, "ronpo_policy_pair_mode": "best_vs_adversary", "seed": 42, "source_dataset": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", "status": "completed", "train_metrics_exists": true, "trainer_state_exists": true}`
Qwen3-8B-scale-up-IFEval-diagnostic-541-prompts
Intended use: reproducibility and evaluation for the RONPO paper. This
checkpoint is not intended as a production assistant.

9
all_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 0.06652991073903643,
"total_flos": 0.0,
"train_loss": 1.094520975748698,
"train_runtime": 689.8738,
"train_samples": 18037,
"train_samples_per_second": 1.739,
"train_steps_per_second": 0.109
}

89
chat_template.jinja Normal file
View File

@@ -0,0 +1,89 @@
{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0].role == 'system' %}
{{- messages[0].content + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0].role == 'system' %}
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endfor %}
{%- for message in messages %}
{%- if message.content is string %}
{%- set content = message.content %}
{%- else %}
{%- set content = '' %}
{%- endif %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- set reasoning_content = '' %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- else %}
{%- if '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
{%- endif %}
{%- endif %}
{%- if loop.index0 > ns.last_query_index %}
{%- if loop.last or (not loop.last and reasoning_content) %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if (loop.first and content) or (not loop.first) %}
{{- '\n' }}
{%- endif %}
{%- if tool_call.function %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n{"name": "' }}
{{- tool_call.name }}
{{- '", "arguments": ' }}
{%- if tool_call.arguments is string %}
{{- tool_call.arguments }}
{%- else %}
{{- tool_call.arguments | tojson }}
{%- endif %}
{{- '}\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' }}
{{- content }}
{{- '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- if enable_thinking is defined and enable_thinking is false %}
{{- '<think>\n\n</think>\n\n' }}
{%- endif %}
{%- endif %}

71
config.json Normal file
View File

@@ -0,0 +1,71 @@
{
"architectures": [
"Qwen3ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": null,
"dtype": "bfloat16",
"eos_token_id": 151645,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 12288,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 40960,
"max_window_layers": 36,
"model_type": "qwen3",
"num_attention_heads": 32,
"num_hidden_layers": 36,
"num_key_value_heads": 8,
"pad_token_id": 151643,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"rope_theta": 1000000,
"rope_type": "default"
},
"sliding_window": null,
"tie_word_embeddings": false,
"transformers_version": "5.13.0",
"use_cache": true,
"use_sliding_window": false,
"vocab_size": 151936
}

54
config.yaml Normal file
View File

@@ -0,0 +1,54 @@
model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218
torch_dtype: null
attn_implementation: eager
dataset_mixer:
/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/full_expect_bestadv_sample1: 1.0
dataset_splits:
- train
- test
preprocessing_num_workers: 12
bf16: true
loss_type: ronpo
max_history_t: 1
history_weights: [1.0]
ronpo_alpha: 1.0
ronpo_tau: 0.05
ronpo_target_column: ronpo_target
reference_anchor_weight: 0
preference_sft_weight: 0.005
beta: 10
learning_rate: 5.0000000000e-07
lr_scheduler_type: cosine
warmup_ratio: 0.1
optim: adamw_torch
weight_decay: 0.0
max_grad_norm: 1.0
seed: 42
gradient_accumulation_steps: 16
gradient_checkpointing: true
gradient_checkpointing_kwargs:
use_reentrant: false
num_train_epochs: 1
max_steps: 75
per_device_train_batch_size: 1
per_device_eval_batch_size: 1
max_length: 2048
max_prompt_length: 1800
do_eval: false
eval_strategy: "no"
eval_steps: 500
logging_steps: 5
log_level: info
generate_during_eval: false
metric_for_best_model: eval_loss
load_best_model_at_end: false
save_strategy: "steps"
save_steps: 1000
save_total_limit: 1
save_only_model: true
save_safetensors: true
push_to_hub: false
report_to:
- wandb
output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3
run_name: rev-q3-ronpo-full_expect-bestadv_sample1-s42-ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"do_sample": true,
"eos_token_id": [
151645,
151643
],
"pad_token_id": 151643,
"temperature": 0.6,
"top_k": 20,
"top_p": 0.95,
"transformers_version": "5.13.0"
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7c1dc4f346065ac2f9498fd9ad3bb4103e54adc508de8a0ed8058591a48232bd
size 16381517208

18
run_status.json Normal file
View File

@@ -0,0 +1,18 @@
{
"method": "ronpo-ablation",
"mode": "full_expect",
"pair_variant": "bestadv_sample1",
"ronpo_policy_pair_mode": "best_vs_adversary",
"adversary_selection": "sample",
"seed": 42,
"status": "completed",
"returncode": 0,
"completed_at": "2026-07-11T23:03:13Z",
"source_dataset": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle",
"dataset": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/full_expect_bestadv_sample1",
"pairs": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/pairs/full_expect_bestadv_sample1",
"output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3",
"config": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ronpo_full_expect_s42_ronpo-bestadv-20260712-0750-r3-sample1_lr5e7_a1_s75_sft5e3/config.yaml",
"train_metrics_exists": true,
"trainer_state_exists": true
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
size 11422650

30
tokenizer_config.json Normal file
View File

@@ -0,0 +1,30 @@
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"is_local": true,
"local_files_only": false,
"model_max_length": 2048,
"pad_token": "<|endoftext|>",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}

9
train_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 0.06652991073903643,
"total_flos": 0.0,
"train_loss": 1.094520975748698,
"train_runtime": 689.8738,
"train_samples": 18037,
"train_samples_per_second": 1.739,
"train_steps_per_second": 0.109
}

403
trainer_state.json Normal file
View File

@@ -0,0 +1,403 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.06652991073903643,
"eval_steps": 500,
"global_step": 75,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.004435327382602428,
"grad_norm": 14.0,
"learning_rate": 2.5e-07,
"logits/chosen": -2.144705295562744,
"logits/rejected": -2.0726897716522217,
"logps/chosen": -0.2952641248703003,
"logps/rejected": -0.3042605519294739,
"loss": 0.9013038873672485,
"loss/core": 0.8998275995254517,
"loss/preference_sft": 0.2952641248703003,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 2.2125720977783203,
"rewards/margins": 1.2788416147232056,
"rewards/rejected": 0.9337307810783386,
"ronpo/logit": 0.1278841495513916,
"ronpo/residual": -0.8721157908439636,
"ronpo/residual_abs": 0.9368273615837097,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 5
},
{
"epoch": 0.008870654765204856,
"grad_norm": 210.0,
"learning_rate": 4.997252228714278e-07,
"logits/chosen": -2.165788412094116,
"logits/rejected": -2.1329445838928223,
"logps/chosen": -0.24870172142982483,
"logps/rejected": -0.28110265731811523,
"loss": 0.9554009437561035,
"loss/core": 0.9541573524475098,
"loss/preference_sft": 0.24870172142982483,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 3.0928869247436523,
"rewards/margins": 1.3761483430862427,
"rewards/rejected": 1.7167381048202515,
"ronpo/logit": 0.13761483132839203,
"ronpo/residual": -0.862385094165802,
"ronpo/residual_abs": 0.9450987577438354,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 10
},
{
"epoch": 0.013305982147807285,
"grad_norm": 11.3125,
"learning_rate": 4.9017129689421e-07,
"logits/chosen": -2.123521089553833,
"logits/rejected": -2.0528640747070312,
"logps/chosen": -0.30137166380882263,
"logps/rejected": -0.2893705666065216,
"loss": 0.9385660290718079,
"loss/core": 0.937059223651886,
"loss/preference_sft": 0.30137166380882263,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 1.6111516952514648,
"rewards/margins": 0.4742206931114197,
"rewards/rejected": 1.13693106174469,
"ronpo/logit": 0.04742206260561943,
"ronpo/residual": -0.9525778889656067,
"ronpo/residual_abs": 0.9525778889656067,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 15
},
{
"epoch": 0.017741309530409713,
"grad_norm": 12.5,
"learning_rate": 4.6747659310219757e-07,
"logits/chosen": -2.2110671997070312,
"logits/rejected": -2.0166139602661133,
"logps/chosen": -0.2711886465549469,
"logps/rejected": -0.29270288348197937,
"loss": 1.0191020965576172,
"loss/core": 1.0177462100982666,
"loss/preference_sft": 0.2711886465549469,
"rewards/accuracies": 0.8125,
"rewards/chosen": 2.056633949279785,
"rewards/margins": 0.49368467926979065,
"rewards/rejected": 1.5629494190216064,
"ronpo/logit": 0.04936846345663071,
"ronpo/residual": -0.9506314992904663,
"ronpo/residual_abs": 0.9506314992904663,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 20
},
{
"epoch": 0.02217663691301214,
"grad_norm": 6.59375,
"learning_rate": 4.3288283381591725e-07,
"logits/chosen": -2.171949625015259,
"logits/rejected": -2.0527703762054443,
"logps/chosen": -0.28833886981010437,
"logps/rejected": -0.29443103075027466,
"loss": 0.9356898069381714,
"loss/core": 0.934248149394989,
"loss/preference_sft": 0.28833886981010437,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 1.5931298732757568,
"rewards/margins": 0.5489386320114136,
"rewards/rejected": 1.0441911220550537,
"ronpo/logit": 0.0548938624560833,
"ronpo/residual": -0.945106029510498,
"ronpo/residual_abs": 0.945106029510498,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 25
},
{
"epoch": 0.02661196429561457,
"grad_norm": 49.75,
"learning_rate": 3.882827885312998e-07,
"logits/chosen": -2.277024269104004,
"logits/rejected": -2.2178688049316406,
"logps/chosen": -0.30705174803733826,
"logps/rejected": -0.3033962845802307,
"loss": 1.0691256523132324,
"loss/core": 1.0675902366638184,
"loss/preference_sft": 0.30705174803733826,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 2.114480495452881,
"rewards/margins": 1.2642418146133423,
"rewards/rejected": 0.8502384424209595,
"ronpo/logit": 0.126424178481102,
"ronpo/residual": -0.8735758662223816,
"ronpo/residual_abs": 0.9844897389411926,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 30
},
{
"epoch": 0.031047291678216998,
"grad_norm": 19.75,
"learning_rate": 3.361167125710832e-07,
"logits/chosen": -2.1667983531951904,
"logits/rejected": -2.209552764892578,
"logps/chosen": -0.29950323700904846,
"logps/rejected": -0.3074646592140198,
"loss": 0.9153361320495605,
"loss/core": 0.9138385057449341,
"loss/preference_sft": 0.29950323700904846,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 1.2793786525726318,
"rewards/margins": 0.6898711919784546,
"rewards/rejected": 0.5895074009895325,
"ronpo/logit": 0.06898711621761322,
"ronpo/residual": -0.9310129284858704,
"ronpo/residual_abs": 0.9501181840896606,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 35
},
{
"epoch": 0.035482619060819426,
"grad_norm": 14.5,
"learning_rate": 2.7923883049302066e-07,
"logits/chosen": -2.0911293029785156,
"logits/rejected": -1.996106743812561,
"logps/chosen": -0.2962394654750824,
"logps/rejected": -0.2883327603340149,
"loss": 1.2223423719406128,
"loss/core": 1.2208611965179443,
"loss/preference_sft": 0.2962394654750824,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 2.883702516555786,
"rewards/margins": 1.178934097290039,
"rewards/rejected": 1.7047685384750366,
"ronpo/logit": 0.11789341270923615,
"ronpo/residual": -0.882106602191925,
"ronpo/residual_abs": 0.9954751133918762,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 40
},
{
"epoch": 0.03991794644342186,
"grad_norm": 17.5,
"learning_rate": 2.2076116950697937e-07,
"logits/chosen": -2.2753586769104004,
"logits/rejected": -2.194331645965576,
"logps/chosen": -0.2942467927932739,
"logps/rejected": -0.2941977381706238,
"loss": 0.960641086101532,
"loss/core": 0.959169864654541,
"loss/preference_sft": 0.2942467927932739,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 2.3440639972686768,
"rewards/margins": 1.245705008506775,
"rewards/rejected": 1.0983591079711914,
"ronpo/logit": 0.12457048892974854,
"ronpo/residual": -0.8754295110702515,
"ronpo/residual_abs": 0.9467538595199585,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 45
},
{
"epoch": 0.04435327382602428,
"grad_norm": 6.03125,
"learning_rate": 1.6388328742891678e-07,
"logits/chosen": -2.3468501567840576,
"logits/rejected": -2.2160422801971436,
"logps/chosen": -0.2907429337501526,
"logps/rejected": -0.299108624458313,
"loss": 1.6810009479522705,
"loss/core": 1.6795470714569092,
"loss/preference_sft": 0.2907429337501526,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": 1.491801381111145,
"rewards/margins": -0.29926931858062744,
"rewards/rejected": 1.7910706996917725,
"ronpo/logit": -0.029926935210824013,
"ronpo/residual": -1.0299268960952759,
"ronpo/residual_abs": 1.0562489032745361,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 50
},
{
"epoch": 0.048788601208626714,
"grad_norm": 5.1875,
"learning_rate": 1.1171721146870014e-07,
"logits/chosen": -2.2220540046691895,
"logits/rejected": -2.159651279449463,
"logps/chosen": -0.29590311646461487,
"logps/rejected": -0.3085595369338989,
"loss": 1.2529202699661255,
"loss/core": 1.2514407634735107,
"loss/preference_sft": 0.29590311646461487,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 1.7223021984100342,
"rewards/margins": 0.2412434071302414,
"rewards/rejected": 1.4810588359832764,
"ronpo/logit": 0.02412433736026287,
"ronpo/residual": -0.9758756756782532,
"ronpo/residual_abs": 1.0081208944320679,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 55
},
{
"epoch": 0.05322392859122914,
"grad_norm": 121.0,
"learning_rate": 6.711716618408281e-08,
"logits/chosen": -1.9089536666870117,
"logits/rejected": -1.977799654006958,
"logps/chosen": -0.3068787455558777,
"logps/rejected": -0.29860004782676697,
"loss": 1.5921037197113037,
"loss/core": 1.590569257736206,
"loss/preference_sft": 0.3068787455558777,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 3.075084924697876,
"rewards/margins": 1.1153666973114014,
"rewards/rejected": 1.9597183465957642,
"ronpo/logit": 0.1115366593003273,
"ronpo/residual": -0.8884633183479309,
"ronpo/residual_abs": 0.9980802536010742,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 60
},
{
"epoch": 0.05765925597383157,
"grad_norm": 25.25,
"learning_rate": 3.2523406897802444e-08,
"logits/chosen": -2.136443614959717,
"logits/rejected": -2.083848476409912,
"logps/chosen": -0.2734604477882385,
"logps/rejected": -0.28085511922836304,
"loss": 0.8904159665107727,
"loss/core": 0.88904869556427,
"loss/preference_sft": 0.2734604477882385,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 1.9123761653900146,
"rewards/margins": 0.9041536450386047,
"rewards/rejected": 1.0082225799560547,
"ronpo/logit": 0.09041537344455719,
"ronpo/residual": -0.909584641456604,
"ronpo/residual_abs": 0.93060702085495,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 65
},
{
"epoch": 0.062094583356433995,
"grad_norm": 8.1875,
"learning_rate": 9.828703105789981e-09,
"logits/chosen": -2.1384823322296143,
"logits/rejected": -2.0494384765625,
"logps/chosen": -0.29097050428390503,
"logps/rejected": -0.3023843765258789,
"loss": 0.8998498916625977,
"loss/core": 0.8983950614929199,
"loss/preference_sft": 0.29097050428390503,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 1.2245758771896362,
"rewards/margins": 0.5744353532791138,
"rewards/rejected": 0.6501404643058777,
"ronpo/logit": 0.057443536818027496,
"ronpo/residual": -0.9425565004348755,
"ronpo/residual_abs": 0.9425565004348755,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 70
},
{
"epoch": 0.06652991073903643,
"grad_norm": 59.25,
"learning_rate": 2.7477712857215675e-10,
"logits/chosen": -2.183204412460327,
"logits/rejected": -2.0384721755981445,
"logps/chosen": -0.27401086688041687,
"logps/rejected": -0.27478593587875366,
"loss": 1.1840168237686157,
"loss/core": 1.182646632194519,
"loss/preference_sft": 0.27401086688041687,
"rewards/accuracies": 0.875,
"rewards/chosen": 1.6213176250457764,
"rewards/margins": 0.029666978865861893,
"rewards/rejected": 1.5916506052017212,
"ronpo/logit": 0.002966696862131357,
"ronpo/residual": -0.9970332980155945,
"ronpo/residual_abs": 0.9970332980155945,
"ronpo/target": 1.0,
"ronpo/target_abs": 1.0,
"ronpo/weight": 1.0,
"ronpo/weight_max": 1.0,
"step": 75
},
{
"epoch": 0.06652991073903643,
"step": 75,
"total_flos": 0.0,
"train_loss": 1.094520975748698,
"train_runtime": 689.8738,
"train_samples_per_second": 1.739,
"train_steps_per_second": 0.109
}
],
"logging_steps": 5,
"max_steps": 75,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 1000,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b68d7e7de68f2b56eab105f77308db7837d47e25f98b0ef6a9e7b3a039a213da
size 7185