promotion/qwen3-8b-inpo-avg-eta0p005-s42
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: promotion/qwen3-8b-inpo-avg-eta0p005-s42
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-08-05 11:49:16 +08:00
commit ff25367b5c
14 changed files with 622 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

25
README.md Normal file
View File

@@ -0,0 +1,25 @@
---
library_name: transformers
base_model: Qwen/Qwen3-8B
tags:
- ronpo
- mnpo
- qwen3
- preference-optimization
---
# qwen3-8b-inpo-avg-eta0p005-s42
Research checkpoint for the RONPO AAAI revision experiments.
- Method: INPO-avg eta=0.005
- Base model: `Qwen/Qwen3-8B` with non-thinking generation protocol
- Seed: 42
- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/inpo_avg_eta0p005_s42_nhn-b200-q3-seq-07101430-inpo`
- Uploaded at UTC: 2026-07-10T16:05:55Z
- Run status metadata: `not found`
Qwen3-8B INPO baseline on the averaged three-reward oracle.
Intended use: reproducibility and evaluation for the RONPO paper. This
checkpoint is not intended as a production assistant.

9
all_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 1.0,
"total_flos": 0.0,
"train_loss": 9980.011584974314,
"train_runtime": 3282.8375,
"train_samples": 18644,
"train_samples_per_second": 5.679,
"train_steps_per_second": 0.044
}

89
chat_template.jinja Normal file
View File

@@ -0,0 +1,89 @@
{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0].role == 'system' %}
{{- messages[0].content + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0].role == 'system' %}
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endfor %}
{%- for message in messages %}
{%- if message.content is string %}
{%- set content = message.content %}
{%- else %}
{%- set content = '' %}
{%- endif %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- set reasoning_content = '' %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- else %}
{%- if '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
{%- endif %}
{%- endif %}
{%- if loop.index0 > ns.last_query_index %}
{%- if loop.last or (not loop.last and reasoning_content) %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if (loop.first and content) or (not loop.first) %}
{{- '\n' }}
{%- endif %}
{%- if tool_call.function %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n{"name": "' }}
{{- tool_call.name }}
{{- '", "arguments": ' }}
{%- if tool_call.arguments is string %}
{{- tool_call.arguments }}
{%- else %}
{{- tool_call.arguments | tojson }}
{%- endif %}
{{- '}\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' }}
{{- content }}
{{- '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- if enable_thinking is defined and enable_thinking is false %}
{{- '<think>\n\n</think>\n\n' }}
{%- endif %}
{%- endif %}

71
config.json Normal file
View File

@@ -0,0 +1,71 @@
{
"architectures": [
"Qwen3ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": null,
"dtype": "bfloat16",
"eos_token_id": 151645,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 12288,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 40960,
"max_window_layers": 36,
"model_type": "qwen3",
"num_attention_heads": 32,
"num_hidden_layers": 36,
"num_key_value_heads": 8,
"pad_token_id": 151643,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"rope_theta": 1000000,
"rope_type": "default"
},
"sliding_window": null,
"tie_word_embeddings": false,
"transformers_version": "5.13.0",
"use_cache": true,
"use_sliding_window": false,
"vocab_size": 151936
}

50
config.yaml Normal file
View File

@@ -0,0 +1,50 @@
model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218
torch_dtype: null
attn_implementation: eager
dataset_mixer:
/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/precomputed/avg_oracle: 1.0
dataset_splits:
- train
- test
preprocessing_num_workers: 12
bf16: true
loss_type: inpo
eta: 0.005
ratio: 0.3333
max_history_t: 1
history_weights: [1.0]
beta: 10
learning_rate: 5.0e-7
lr_scheduler_type: cosine
warmup_ratio: 0.03
optim: adamw_torch
weight_decay: 0.0
seed: 42
gradient_accumulation_steps: 16
gradient_checkpointing: true
gradient_checkpointing_kwargs:
use_reentrant: false
num_train_epochs: 1
per_device_train_batch_size: 1
per_device_eval_batch_size: 1
max_length: 2048
max_prompt_length: 1800
do_eval: true
eval_strategy: "steps"
eval_steps: 500
logging_steps: 10
log_level: info
generate_during_eval: false
metric_for_best_model: eval_loss
load_best_model_at_end: false
save_strategy: "steps"
save_steps: 1000
save_total_limit: 1
save_only_model: true
save_safetensors: true
push_to_hub: false
report_to:
- wandb
output_dir: /NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/inpo_avg_eta0p005_s42_nhn-b200-q3-seq-07101430-inpo
run_name: rev-q3-inpo-eta0p005-s42-nhn-b200-q3-seq-07101430-inpo

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"do_sample": true,
"eos_token_id": [
151645,
151643
],
"pad_token_id": 151643,
"temperature": 0.6,
"top_k": 20,
"top_p": 0.95,
"transformers_version": "5.13.0"
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:74908e815fb17502457849a0164eafe0833111c5e43b68c9150d4a7ae4525170
size 16381517208

13
run_status.json Normal file
View File

@@ -0,0 +1,13 @@
{
"method": "inpo-avg",
"loss_type": "inpo",
"eta": 0.005,
"seed": 42,
"status": "completed",
"returncode": 0,
"completed_at": "2026-07-10T16:03:33Z",
"dataset": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/precomputed/avg_oracle",
"output_dir": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/inpo_avg_eta0p005_s42_nhn-b200-q3-seq-07101430-inpo",
"train_metrics_exists": true,
"trainer_state_exists": true
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
size 11422650

30
tokenizer_config.json Normal file
View File

@@ -0,0 +1,30 @@
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"is_local": true,
"local_files_only": false,
"model_max_length": 2048,
"pad_token": "<|endoftext|>",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}

9
train_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 1.0,
"total_flos": 0.0,
"train_loss": 9980.011584974314,
"train_runtime": 3282.8375,
"train_samples": 18644,
"train_samples_per_second": 5.679,
"train_steps_per_second": 0.044
}

269
trainer_state.json Normal file
View File

@@ -0,0 +1,269 @@
{
"best_global_step": 146,
"best_metric": 9972.7021484375,
"best_model_checkpoint": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/inpo_avg_eta0p005_s42_nhn-b200-q3-seq-07101430-inpo/checkpoint-146",
"epoch": 1.0,
"eval_steps": 500,
"global_step": 146,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.06864006864006864,
"grad_norm": 20387.03686881509,
"learning_rate": 4.990077890125363e-07,
"logits/chosen": -0.796875,
"logits/rejected": -0.8046875,
"logps/chosen": -16.0,
"logps/rejected": -15.9375,
"loss": 9979.939453125,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -153.50888061523438,
"rewards/margins": 1.0440444946289062,
"rewards/rejected": -154.55291748046875,
"step": 10
},
{
"epoch": 0.13728013728013727,
"grad_norm": 16463.56555342243,
"learning_rate": 4.879356673988089e-07,
"logits/chosen": -0.87109375,
"logits/rejected": -0.84765625,
"logps/chosen": -15.8125,
"logps/rejected": -15.875,
"loss": 9975.23828125,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -153.51373291015625,
"rewards/margins": 1.252532958984375,
"rewards/rejected": -154.76626586914062,
"step": 20
},
{
"epoch": 0.2059202059202059,
"grad_norm": 28044.862093220032,
"learning_rate": 4.6510039481503485e-07,
"logits/chosen": -0.74609375,
"logits/rejected": -0.7421875,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 9996.552734375,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -155.00054931640625,
"rewards/margins": 0.214385986328125,
"rewards/rejected": -155.21493530273438,
"step": 30
},
{
"epoch": 0.27456027456027454,
"grad_norm": 19615.09853150976,
"learning_rate": 4.3163090985954074e-07,
"logits/chosen": -0.734375,
"logits/rejected": -0.74609375,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 9994.1796875,
"rewards/accuracies": 0.53125,
"rewards/chosen": -155.23875427246094,
"rewards/margins": 0.32525634765625,
"rewards/rejected": -155.56399536132812,
"step": 40
},
{
"epoch": 0.3432003432003432,
"grad_norm": 18075.561008642915,
"learning_rate": 3.891818892301304e-07,
"logits/chosen": -0.71484375,
"logits/rejected": -0.71875,
"logps/chosen": -16.125,
"logps/rejected": -16.125,
"loss": 9973.58203125,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -154.60614013671875,
"rewards/margins": 1.3658523559570312,
"rewards/rejected": -155.9720001220703,
"step": 50
},
{
"epoch": 0.4118404118404118,
"grad_norm": 21829.673229552107,
"learning_rate": 3.3985194320937815e-07,
"logits/chosen": -0.6953125,
"logits/rejected": -0.6875,
"logps/chosen": -16.125,
"logps/rejected": -16.125,
"loss": 9989.525390625,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -156.18487548828125,
"rewards/margins": 0.5427627563476562,
"rewards/rejected": -156.72763061523438,
"step": 60
},
{
"epoch": 0.4804804804804805,
"grad_norm": 19354.982747194757,
"learning_rate": 2.8607986379820664e-07,
"logits/chosen": -0.66015625,
"logits/rejected": -0.6484375,
"logps/chosen": -16.0,
"logps/rejected": -16.125,
"loss": 9972.458984375,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -155.47698974609375,
"rewards/margins": 1.410614013671875,
"rewards/rejected": -156.88760375976562,
"step": 70
},
{
"epoch": 0.5491205491205491,
"grad_norm": 23696.769639901566,
"learning_rate": 2.3052405482064919e-07,
"logits/chosen": -0.65625,
"logits/rejected": -0.67578125,
"logps/chosen": -16.125,
"logps/rejected": -16.0,
"loss": 9998.822265625,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -155.87271118164062,
"rewards/margins": 0.087371826171875,
"rewards/rejected": -155.9600830078125,
"step": 80
},
{
"epoch": 0.6177606177606177,
"grad_norm": 40006.950593012094,
"learning_rate": 1.7593110477859152e-07,
"logits/chosen": -0.67578125,
"logits/rejected": -0.671875,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 9985.57421875,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -154.62530517578125,
"rewards/margins": 0.742950439453125,
"rewards/rejected": -155.36825561523438,
"step": 90
},
{
"epoch": 0.6864006864006864,
"grad_norm": 18244.96636423906,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.62890625,
"logits/rejected": -0.6171875,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 9980.126953125,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -154.44403076171875,
"rewards/margins": 1.02630615234375,
"rewards/rejected": -155.47032165527344,
"step": 100
},
{
"epoch": 0.7550407550407551,
"grad_norm": 27502.551311382453,
"learning_rate": 8.024869116091879e-08,
"logits/chosen": -0.6484375,
"logits/rejected": -0.62890625,
"logps/chosen": -16.0,
"logps/rejected": -16.125,
"loss": 9976.1513671875,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -155.1019287109375,
"rewards/margins": 1.234588623046875,
"rewards/rejected": -156.33651733398438,
"step": 110
},
{
"epoch": 0.8236808236808236,
"grad_norm": 19647.533994105936,
"learning_rate": 4.388960991455998e-08,
"logits/chosen": -0.58984375,
"logits/rejected": -0.59375,
"logps/chosen": -16.125,
"logps/rejected": -16.125,
"loss": 9963.115234375,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -154.8621826171875,
"rewards/margins": 1.900604248046875,
"rewards/rejected": -156.76278686523438,
"step": 120
},
{
"epoch": 0.8923208923208923,
"grad_norm": 20937.34630051602,
"learning_rate": 1.7720289882128092e-08,
"logits/chosen": -0.62109375,
"logits/rejected": -0.609375,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 9983.845703125,
"rewards/accuracies": 0.53125,
"rewards/chosen": -155.11953735351562,
"rewards/margins": 0.8319244384765625,
"rewards/rejected": -155.9514617919922,
"step": 130
},
{
"epoch": 0.960960960960961,
"grad_norm": 20078.25155988699,
"learning_rate": 3.0344995250326245e-09,
"logits/chosen": -0.578125,
"logits/rejected": -0.58984375,
"logps/chosen": -16.0,
"logps/rejected": -16.0,
"loss": 9975.1064453125,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -155.063720703125,
"rewards/margins": 1.2694091796875,
"rewards/rejected": -156.3331298828125,
"step": 140
},
{
"epoch": 1.0,
"eval_logits/chosen": -0.63671875,
"eval_logits/rejected": -0.60546875,
"eval_logps/chosen": -15.9375,
"eval_logps/rejected": -16.0,
"eval_loss": 9972.7021484375,
"eval_rewards/accuracies": 0.5526315569877625,
"eval_rewards/chosen": -155.34686279296875,
"eval_rewards/margins": 1.3776253461837769,
"eval_rewards/rejected": -156.7244873046875,
"eval_runtime": 68.5896,
"eval_samples_per_second": 8.806,
"eval_steps_per_second": 1.108,
"step": 146
},
{
"epoch": 1.0,
"step": 146,
"total_flos": 0.0,
"train_loss": 9980.011584974314,
"train_runtime": 3282.8375,
"train_samples_per_second": 5.679,
"train_steps_per_second": 0.044
}
],
"logging_steps": 10,
"max_steps": 146,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 1000,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:80c3f05952d453eb503ee8afb27b7b546fef9a5f4b71a245a8140a22ce0fbc25
size 8401