promotion/qwen3-8b-ipo-avg-beta0p01-s42
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: promotion/qwen3-8b-ipo-avg-beta0p01-s42
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-08-06 12:53:16 +08:00
commit 54dea626c0
14 changed files with 627 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

25
README.md Normal file
View File

@@ -0,0 +1,25 @@
---
library_name: transformers
base_model: Qwen/Qwen3-8B
tags:
- ronpo
- mnpo
- qwen3
- preference-optimization
---
# qwen3-8b-ipo-avg-beta0p01-s42
Research checkpoint for the RONPO AAAI revision experiments.
- Method: IPO-avg beta=0.01
- Base model: `Qwen/Qwen3-8B` with non-thinking generation protocol
- Seed: 42
- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/ipo_avg_beta0p01_s42_nhn-b200-q3-seq-07101430-ipo`
- Uploaded at UTC: 2026-07-11T03:10:12Z
- Run status metadata: `not found`
Qwen3-8B IPO baseline on the averaged three-reward oracle.
Intended use: reproducibility and evaluation for the RONPO paper. This
checkpoint is not intended as a production assistant.

9
all_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 1.0,
"total_flos": 0.0,
"train_loss": 2490.3512815710615,
"train_runtime": 3284.6395,
"train_samples": 18644,
"train_samples_per_second": 5.676,
"train_steps_per_second": 0.044
}

89
chat_template.jinja Normal file
View File

@@ -0,0 +1,89 @@
{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0].role == 'system' %}
{{- messages[0].content + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0].role == 'system' %}
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endfor %}
{%- for message in messages %}
{%- if message.content is string %}
{%- set content = message.content %}
{%- else %}
{%- set content = '' %}
{%- endif %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- set reasoning_content = '' %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- else %}
{%- if '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
{%- endif %}
{%- endif %}
{%- if loop.index0 > ns.last_query_index %}
{%- if loop.last or (not loop.last and reasoning_content) %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if (loop.first and content) or (not loop.first) %}
{{- '\n' }}
{%- endif %}
{%- if tool_call.function %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n{"name": "' }}
{{- tool_call.name }}
{{- '", "arguments": ' }}
{%- if tool_call.arguments is string %}
{{- tool_call.arguments }}
{%- else %}
{{- tool_call.arguments | tojson }}
{%- endif %}
{{- '}\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' }}
{{- content }}
{{- '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- if enable_thinking is defined and enable_thinking is false %}
{{- '<think>\n\n</think>\n\n' }}
{%- endif %}
{%- endif %}

71
config.json Normal file
View File

@@ -0,0 +1,71 @@
{
"architectures": [
"Qwen3ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": null,
"dtype": "bfloat16",
"eos_token_id": 151645,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 12288,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 40960,
"max_window_layers": 36,
"model_type": "qwen3",
"num_attention_heads": 32,
"num_hidden_layers": 36,
"num_key_value_heads": 8,
"pad_token_id": 151643,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"rope_theta": 1000000,
"rope_type": "default"
},
"sliding_window": null,
"tie_word_embeddings": false,
"transformers_version": "5.13.0",
"use_cache": true,
"use_sliding_window": false,
"vocab_size": 151936
}

53
config.yaml Normal file
View File

@@ -0,0 +1,53 @@
model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218
torch_dtype: null
attn_implementation: eager
dataset_mixer:
/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/precomputed/avg_oracle: 1.0
dataset_splits:
- train
- test
preprocessing_num_workers: 12
bf16: true
loss_type: ipo
dpo_beta: 0.01
simpo_beta: 2.5
simpo_gamma: 1.0
eta: 0.0075
ratio: 0.3333
max_history_t: 1
history_weights: [1.0]
beta: 10
learning_rate: 5.0e-7
lr_scheduler_type: cosine
warmup_ratio: 0.1
optim: adamw_torch
weight_decay: 0.0
seed: 42
gradient_accumulation_steps: 16
gradient_checkpointing: true
gradient_checkpointing_kwargs:
use_reentrant: false
num_train_epochs: 1
per_device_train_batch_size: 1
per_device_eval_batch_size: 1
max_length: 2048
max_prompt_length: 1800
do_eval: true
eval_strategy: "steps"
eval_steps: 500
logging_steps: 10
log_level: info
generate_during_eval: false
metric_for_best_model: eval_loss
load_best_model_at_end: false
save_strategy: "steps"
save_steps: 1000
save_total_limit: 1
save_only_model: true
save_safetensors: true
push_to_hub: false
report_to:
- wandb
output_dir: /NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/ipo_avg_beta0p01_s42_nhn-b200-q3-seq-07101430-ipo
run_name: rev-q3-ipo-b0p01-s42-nhn-b200-q3-seq-07101430-ipo

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"do_sample": true,
"eos_token_id": [
151645,
151643
],
"pad_token_id": 151643,
"temperature": 0.6,
"top_k": 20,
"top_p": 0.95,
"transformers_version": "5.13.0"
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f920002de84073c192a44d179183154a1601e73b7b62b70b2531da96330900af
size 16381517208

15
run_status.json Normal file
View File

@@ -0,0 +1,15 @@
{
"method": "ipo",
"loss_type": "ipo",
"seed": 42,
"beta": 0.01,
"backbone": "Qwen/Qwen3-8B",
"status": "completed",
"returncode": 0,
"completed_at": "2026-07-11T03:08:02Z",
"dataset": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/precomputed/avg_oracle",
"output_dir": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/ipo_avg_beta0p01_s42_nhn-b200-q3-seq-07101430-ipo",
"config": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/ipo_avg_beta0p01_s42_nhn-b200-q3-seq-07101430-ipo/config.yaml",
"train_metrics_exists": true,
"trainer_state_exists": true
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
size 11422650

30
tokenizer_config.json Normal file
View File

@@ -0,0 +1,30 @@
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"is_local": true,
"local_files_only": false,
"model_max_length": 2048,
"pad_token": "<|endoftext|>",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}

9
train_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 1.0,
"total_flos": 0.0,
"train_loss": 2490.3512815710615,
"train_runtime": 3284.6395,
"train_samples": 18644,
"train_samples_per_second": 5.676,
"train_steps_per_second": 0.044
}

269
trainer_state.json Normal file
View File

@@ -0,0 +1,269 @@
{
"best_global_step": 146,
"best_metric": 2487.621826171875,
"best_model_checkpoint": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/ipo_avg_beta0p01_s42_nhn-b200-q3-seq-07101430-ipo/checkpoint-146",
"epoch": 1.0,
"eval_steps": 500,
"global_step": 146,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.06864006864006864,
"grad_norm": 10027.114290420466,
"learning_rate": 3e-07,
"logits/chosen": -0.8125,
"logits/rejected": -0.8203125,
"logps/chosen": -16.0,
"logps/rejected": -15.9375,
"loss": 2489.71826171875,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -153.37606811523438,
"rewards/margins": 1.1065444946289062,
"rewards/rejected": -154.48260498046875,
"step": 10
},
{
"epoch": 0.13728013728013727,
"grad_norm": 8819.949018899022,
"learning_rate": 4.988506452457066e-07,
"logits/chosen": -0.8828125,
"logits/rejected": -0.85546875,
"logps/chosen": -15.8125,
"logps/rejected": -15.875,
"loss": 2487.34423828125,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -153.25982666015625,
"rewards/margins": 1.295501708984375,
"rewards/rejected": -154.55532836914062,
"step": 20
},
{
"epoch": 0.2059202059202059,
"grad_norm": 8211.073831209287,
"learning_rate": 4.860414659112948e-07,
"logits/chosen": -0.76953125,
"logits/rejected": -0.77734375,
"logps/chosen": -16.0,
"logps/rejected": -15.9375,
"loss": 2502.94287109375,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -154.95367431640625,
"rewards/margins": -0.227020263671875,
"rewards/rejected": -154.72665405273438,
"step": 30
},
{
"epoch": 0.27456027456027454,
"grad_norm": 8719.803872679653,
"learning_rate": 4.597220474379125e-07,
"logits/chosen": -0.76171875,
"logits/rejected": -0.7734375,
"logps/chosen": -16.0,
"logps/rejected": -15.9375,
"loss": 2498.837158203125,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -154.68797302246094,
"rewards/margins": 0.18853759765625,
"rewards/rejected": -154.87649536132812,
"step": 40
},
{
"epoch": 0.3432003432003432,
"grad_norm": 9152.672087584928,
"learning_rate": 4.2139882699413613e-07,
"logits/chosen": -0.765625,
"logits/rejected": -0.74609375,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 2489.340087890625,
"rewards/accuracies": 0.5062500238418579,
"rewards/chosen": -154.12176513671875,
"rewards/margins": 1.1353836059570312,
"rewards/rejected": -155.2571563720703,
"step": 50
},
{
"epoch": 0.4118404118404118,
"grad_norm": 14845.378062289448,
"learning_rate": 3.7326529999303633e-07,
"logits/chosen": -0.73046875,
"logits/rejected": -0.734375,
"logps/chosen": -16.0,
"logps/rejected": -15.9375,
"loss": 2498.56591796875,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -155.47003173828125,
"rewards/margins": 0.18729400634765625,
"rewards/rejected": -155.65731811523438,
"step": 60
},
{
"epoch": 0.4804804804804805,
"grad_norm": 10089.737237780628,
"learning_rate": 3.1807647161082797e-07,
"logits/chosen": -0.72265625,
"logits/rejected": -0.69140625,
"logps/chosen": -15.9375,
"logps/rejected": -15.9375,
"loss": 2490.2265625,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -154.46917724609375,
"rewards/margins": 1.055145263671875,
"rewards/rejected": -155.52432250976562,
"step": 70
},
{
"epoch": 0.5491205491205491,
"grad_norm": 13080.69256033714,
"learning_rate": 2.5899116932879534e-07,
"logits/chosen": -0.7109375,
"logits/rejected": -0.71875,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 2498.123046875,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -155.04067993164062,
"rewards/margins": 0.243621826171875,
"rewards/rejected": -155.2843017578125,
"step": 80
},
{
"epoch": 0.6177606177606177,
"grad_norm": 10488.56173199215,
"learning_rate": 1.993912420112756e-07,
"logits/chosen": -0.7265625,
"logits/rejected": -0.7109375,
"logps/chosen": -15.875,
"logps/rejected": -15.875,
"loss": 2492.578369140625,
"rewards/accuracies": 0.53125,
"rewards/chosen": -153.39874267578125,
"rewards/margins": 0.793731689453125,
"rewards/rejected": -154.19247436523438,
"step": 90
},
{
"epoch": 0.6864006864006864,
"grad_norm": 12700.337502115342,
"learning_rate": 1.426879939713322e-07,
"logits/chosen": -0.69140625,
"logits/rejected": -0.6875,
"logps/chosen": -15.875,
"logps/rejected": -15.875,
"loss": 2490.487060546875,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -153.31903076171875,
"rewards/margins": 1.01458740234375,
"rewards/rejected": -154.33360290527344,
"step": 100
},
{
"epoch": 0.7550407550407551,
"grad_norm": 9077.531663523849,
"learning_rate": 9.212693310317479e-08,
"logits/chosen": -0.69140625,
"logits/rejected": -0.69140625,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 2487.44677734375,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -154.6409912109375,
"rewards/margins": 1.328338623046875,
"rewards/rejected": -155.96932983398438,
"step": 110
},
{
"epoch": 0.8236808236808236,
"grad_norm": 8718.070043453483,
"learning_rate": 5.060200865767605e-08,
"logits/chosen": -0.625,
"logits/rejected": -0.62890625,
"logps/chosen": -16.0,
"logps/rejected": -16.125,
"loss": 2481.56396484375,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -153.7762451171875,
"rewards/margins": 1.955291748046875,
"rewards/rejected": -155.73153686523438,
"step": 120
},
{
"epoch": 0.8923208923208923,
"grad_norm": 11737.493049054548,
"learning_rate": 2.0489971081290193e-08,
"logits/chosen": -0.66015625,
"logits/rejected": -0.66015625,
"logps/chosen": -15.9375,
"logps/rejected": -15.9375,
"loss": 2495.81494140625,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": -154.38516235351562,
"rewards/margins": 0.4686431884765625,
"rewards/rejected": -154.8538055419922,
"step": 130
},
{
"epoch": 0.960960960960961,
"grad_norm": 9402.719273365525,
"learning_rate": 3.5143346177878565e-09,
"logits/chosen": -0.640625,
"logits/rejected": -0.62890625,
"logps/chosen": -15.9375,
"logps/rejected": -16.0,
"loss": 2483.26611328125,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -154.122314453125,
"rewards/margins": 1.7264404296875,
"rewards/rejected": -155.8487548828125,
"step": 140
},
{
"epoch": 1.0,
"eval_logits/chosen": -0.65234375,
"eval_logits/rejected": -0.625,
"eval_logps/chosen": -15.9375,
"eval_logps/rejected": -16.0,
"eval_loss": 2487.621826171875,
"eval_rewards/accuracies": 0.5526315569877625,
"eval_rewards/chosen": -154.9027862548828,
"eval_rewards/margins": 1.2624943256378174,
"eval_rewards/rejected": -156.165283203125,
"eval_runtime": 69.2925,
"eval_samples_per_second": 8.717,
"eval_steps_per_second": 1.097,
"step": 146
},
{
"epoch": 1.0,
"step": 146,
"total_flos": 0.0,
"train_loss": 2490.3512815710615,
"train_runtime": 3284.6395,
"train_samples_per_second": 5.676,
"train_steps_per_second": 0.044
}
],
"logging_steps": 10,
"max_steps": 146,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 1000,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:096ce8a681d0524d243f91b47ce7c5438fabe6ce829262b46c0eecd08477eb93
size 8401