promotion/qwen3-8b-htmnpo-skywork-s42
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: promotion/qwen3-8b-htmnpo-skywork-s42
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-08-05 11:27:16 +08:00
commit 567b490a05
14 changed files with 700 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

25
README.md Normal file
View File

@@ -0,0 +1,25 @@
---
library_name: transformers
base_model: Qwen/Qwen3-8B
tags:
- ronpo
- mnpo
- qwen3
- preference-optimization
---
# qwen3-8b-htmnpo-skywork-s42
Research checkpoint for the RONPO AAAI revision experiments.
- Method: HT-MNPO skywork
- Base model: `Qwen/Qwen3-8B` with non-thinking generation protocol
- Seed: 42
- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/train/htmnpo_skywork_s42_nhn-b200-q3-seq-07101430-htmnpo`
- Uploaded at UTC: 2026-07-11T06:32:58Z
- Run status metadata: `not found`
Qwen3-8B HT-MNPO single-oracle baseline.
Intended use: reproducibility and evaluation for the RONPO paper. This
checkpoint is not intended as a production assistant.

9
all_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 1.0,
"total_flos": 0.0,
"train_loss": 0.5071899907229698,
"train_runtime": 3280.3544,
"train_samples": 18640,
"train_samples_per_second": 5.682,
"train_steps_per_second": 0.045
}

89
chat_template.jinja Normal file
View File

@@ -0,0 +1,89 @@
{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0].role == 'system' %}
{{- messages[0].content + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0].role == 'system' %}
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endfor %}
{%- for message in messages %}
{%- if message.content is string %}
{%- set content = message.content %}
{%- else %}
{%- set content = '' %}
{%- endif %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- set reasoning_content = '' %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- else %}
{%- if '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
{%- endif %}
{%- endif %}
{%- if loop.index0 > ns.last_query_index %}
{%- if loop.last or (not loop.last and reasoning_content) %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if (loop.first and content) or (not loop.first) %}
{{- '\n' }}
{%- endif %}
{%- if tool_call.function %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n{"name": "' }}
{{- tool_call.name }}
{{- '", "arguments": ' }}
{%- if tool_call.arguments is string %}
{{- tool_call.arguments }}
{%- else %}
{{- tool_call.arguments | tojson }}
{%- endif %}
{{- '}\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' }}
{{- content }}
{{- '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- if enable_thinking is defined and enable_thinking is false %}
{{- '<think>\n\n</think>\n\n' }}
{%- endif %}
{%- endif %}

71
config.json Normal file
View File

@@ -0,0 +1,71 @@
{
"architectures": [
"Qwen3ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": null,
"dtype": "bfloat16",
"eos_token_id": 151645,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 12288,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 40960,
"max_window_layers": 36,
"model_type": "qwen3",
"num_attention_heads": 32,
"num_hidden_layers": 36,
"num_key_value_heads": 8,
"pad_token_id": 151643,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"rope_theta": 1000000,
"rope_type": "default"
},
"sliding_window": null,
"tie_word_embeddings": false,
"transformers_version": "5.13.0",
"use_cache": true,
"use_sliding_window": false,
"vocab_size": 151936
}

52
config.yaml Normal file
View File

@@ -0,0 +1,52 @@
model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218
torch_dtype: null
attn_implementation: eager
dataset_mixer:
/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/precomputed/ht_skywork: 1.0
dataset_splits:
- train
- test
preprocessing_num_workers: 12
bf16: true
loss_type: ht_mnpo
eta: 0.0075
ratio: 0.3333
max_history_t: 1
history_weights: [1.0]
ht_target_column: ht_target
ht_target_scale: 1.0
beta: 10
learning_rate: 5.0e-7
lr_scheduler_type: cosine
warmup_ratio: 0.1
optim: adamw_torch
weight_decay: 0.0
seed: 42
gradient_accumulation_steps: 16
gradient_checkpointing: true
gradient_checkpointing_kwargs:
use_reentrant: false
num_train_epochs: 1
per_device_train_batch_size: 1
per_device_eval_batch_size: 1
max_length: 2048
max_prompt_length: 1800
do_eval: true
eval_strategy: "steps"
eval_steps: 500
logging_steps: 10
log_level: info
generate_during_eval: false
metric_for_best_model: eval_loss
load_best_model_at_end: false
save_strategy: "steps"
save_steps: 1000
save_total_limit: 1
save_only_model: true
save_safetensors: true
push_to_hub: false
report_to:
- wandb
output_dir: /NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/htmnpo_skywork_s42_nhn-b200-q3-seq-07101430-htmnpo
run_name: rev-q3-htmnpo-skywork-s42-nhn-b200-q3-seq-07101430-htmnpo

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"do_sample": true,
"eos_token_id": [
151645,
151643
],
"pad_token_id": 151643,
"temperature": 0.6,
"top_k": 20,
"top_p": 0.95,
"transformers_version": "5.13.0"
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ba880fa98ab3af3ac68bb720f6294f057ce623041b0d0d0b8b779c224fbd96f5
size 16381517208

14
run_status.json Normal file
View File

@@ -0,0 +1,14 @@
{
"method": "ht-mnpo",
"loss_type": "ht_mnpo",
"objective": "skywork",
"eta": 0.0075,
"seed": 42,
"status": "completed",
"returncode": 0,
"completed_at": "2026-07-11T06:30:45Z",
"dataset": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/precomputed/ht_skywork",
"output_dir": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/htmnpo_skywork_s42_nhn-b200-q3-seq-07101430-htmnpo",
"train_metrics_exists": true,
"trainer_state_exists": true
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
size 11422650

30
tokenizer_config.json Normal file
View File

@@ -0,0 +1,30 @@
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"is_local": true,
"local_files_only": false,
"model_max_length": 2048,
"pad_token": "<|endoftext|>",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}

9
train_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 1.0,
"total_flos": 0.0,
"train_loss": 0.5071899907229698,
"train_runtime": 3280.3544,
"train_samples": 18640,
"train_samples_per_second": 5.682,
"train_steps_per_second": 0.045
}

344
trainer_state.json Normal file
View File

@@ -0,0 +1,344 @@
{
"best_global_step": 146,
"best_metric": 0.09535243362188339,
"best_model_checkpoint": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/htmnpo_skywork_s42_nhn-b200-q3-seq-07101430-htmnpo/checkpoint-146",
"epoch": 1.0,
"eval_steps": 500,
"global_step": 146,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.06866952789699571,
"grad_norm": 658.2609913586693,
"ht_mnpo/logit": -0.07977066189050674,
"ht_mnpo/residual": -0.08727066218852997,
"ht_mnpo/residual_abs": 0.5136820673942566,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 3e-07,
"logits/chosen": -0.81640625,
"logits/rejected": -0.81640625,
"logps/chosen": -15.9375,
"logps/rejected": -15.875,
"loss": 1.032188057899475,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -154.9237518310547,
"rewards/margins": -0.7977066040039062,
"rewards/rejected": -154.1260528564453,
"step": 10
},
{
"epoch": 0.13733905579399142,
"grad_norm": 486.85604769071176,
"ht_mnpo/logit": 0.07697143405675888,
"ht_mnpo/residual": 0.06947144120931625,
"ht_mnpo/residual_abs": 0.46500977873802185,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 4.988506452457066e-07,
"logits/chosen": -0.80859375,
"logits/rejected": -0.83984375,
"logps/chosen": -15.875,
"logps/rejected": -15.875,
"loss": 0.5586695075035095,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -153.60064697265625,
"rewards/margins": 0.76971435546875,
"rewards/rejected": -154.370361328125,
"step": 20
},
{
"epoch": 0.20600858369098712,
"grad_norm": 279.6489736984974,
"ht_mnpo/logit": 0.08701781928539276,
"ht_mnpo/residual": 0.07951782643795013,
"ht_mnpo/residual_abs": 0.48369044065475464,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 4.860414659112948e-07,
"logits/chosen": -0.8359375,
"logits/rejected": -0.8359375,
"logps/chosen": -15.75,
"logps/rejected": -15.75,
"loss": 0.6932045221328735,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -152.07644653320312,
"rewards/margins": 0.87017822265625,
"rewards/rejected": -152.94662475585938,
"step": 30
},
{
"epoch": 0.27467811158798283,
"grad_norm": 405.5407274927664,
"ht_mnpo/logit": 0.11707458645105362,
"ht_mnpo/residual": 0.1095745787024498,
"ht_mnpo/residual_abs": 0.40353816747665405,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 4.597220474379125e-07,
"logits/chosen": -0.82421875,
"logits/rejected": -0.82421875,
"logps/chosen": -15.5,
"logps/rejected": -15.5,
"loss": 0.6164440512657166,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": -149.45849609375,
"rewards/margins": 1.170745849609375,
"rewards/rejected": -150.62924194335938,
"step": 40
},
{
"epoch": 0.34334763948497854,
"grad_norm": 173.47778729238564,
"ht_mnpo/logit": 0.09282531589269638,
"ht_mnpo/residual": 0.08532532304525375,
"ht_mnpo/residual_abs": 0.3633372485637665,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 4.2139882699413613e-07,
"logits/chosen": -0.89453125,
"logits/rejected": -0.921875,
"logps/chosen": -14.9375,
"logps/rejected": -14.9375,
"loss": 0.38052481412887573,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -144.1920166015625,
"rewards/margins": 0.928253173828125,
"rewards/rejected": -145.12026977539062,
"step": 50
},
{
"epoch": 0.41201716738197425,
"grad_norm": 220.4191286143336,
"ht_mnpo/logit": 0.15768662095069885,
"ht_mnpo/residual": 0.15018662810325623,
"ht_mnpo/residual_abs": 0.4016141891479492,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 3.7326529999303633e-07,
"logits/chosen": -0.89453125,
"logits/rejected": -0.91796875,
"logps/chosen": -14.5625,
"logps/rejected": -14.625,
"loss": 0.8031173944473267,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -140.22152709960938,
"rewards/margins": 1.5768661499023438,
"rewards/rejected": -141.7983856201172,
"step": 60
},
{
"epoch": 0.48068669527896996,
"grad_norm": 226.8002386346793,
"ht_mnpo/logit": 0.15667113661766052,
"ht_mnpo/residual": 0.1491711437702179,
"ht_mnpo/residual_abs": 0.4558327794075012,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 3.1807647161082797e-07,
"logits/chosen": -0.890625,
"logits/rejected": -0.8828125,
"logps/chosen": -14.3125,
"logps/rejected": -14.3125,
"loss": 1.1153709888458252,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -136.7957305908203,
"rewards/margins": 1.56671142578125,
"rewards/rejected": -138.36244201660156,
"step": 70
},
{
"epoch": 0.5493562231759657,
"grad_norm": 188.6296821878058,
"ht_mnpo/logit": 0.040885161608457565,
"ht_mnpo/residual": 0.03338516503572464,
"ht_mnpo/residual_abs": 0.35882776975631714,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 2.5899116932879534e-07,
"logits/chosen": -0.83203125,
"logits/rejected": -0.87109375,
"logps/chosen": -14.1875,
"logps/rejected": -14.1875,
"loss": 0.46190714836120605,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -136.6307373046875,
"rewards/margins": 0.40885162353515625,
"rewards/rejected": -137.03958129882812,
"step": 80
},
{
"epoch": 0.6180257510729614,
"grad_norm": 159.63381058216007,
"ht_mnpo/logit": 0.2624649107456207,
"ht_mnpo/residual": 0.2549648880958557,
"ht_mnpo/residual_abs": 0.4282897412776947,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 1.993912420112756e-07,
"logits/chosen": -0.83984375,
"logits/rejected": -0.84375,
"logps/chosen": -14.0625,
"logps/rejected": -14.125,
"loss": 0.8312055468559265,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -134.35748291015625,
"rewards/margins": 2.6246490478515625,
"rewards/rejected": -136.9821319580078,
"step": 90
},
{
"epoch": 0.6866952789699571,
"grad_norm": 217.2628805931693,
"ht_mnpo/logit": 0.059761811047792435,
"ht_mnpo/residual": 0.05226181820034981,
"ht_mnpo/residual_abs": 0.303397536277771,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 1.426879939713322e-07,
"logits/chosen": -0.86328125,
"logits/rejected": -0.875,
"logps/chosen": -13.875,
"logps/rejected": -13.875,
"loss": 0.22050431370735168,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -134.1347198486328,
"rewards/margins": 0.5976181030273438,
"rewards/rejected": -134.73233032226562,
"step": 100
},
{
"epoch": 0.7553648068669528,
"grad_norm": 307.6598272099915,
"ht_mnpo/logit": 0.08397064357995987,
"ht_mnpo/residual": 0.07647065818309784,
"ht_mnpo/residual_abs": 0.3651581406593323,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 9.212693310317479e-08,
"logits/chosen": -0.83203125,
"logits/rejected": -0.84765625,
"logps/chosen": -13.9375,
"logps/rejected": -13.9375,
"loss": 0.46880072355270386,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -133.74819946289062,
"rewards/margins": 0.8397064208984375,
"rewards/rejected": -134.58790588378906,
"step": 110
},
{
"epoch": 0.8240343347639485,
"grad_norm": 101.02657333091942,
"ht_mnpo/logit": 0.02447204664349556,
"ht_mnpo/residual": 0.016972053796052933,
"ht_mnpo/residual_abs": 0.24492737650871277,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 5.060200865767605e-08,
"logits/chosen": -0.80859375,
"logits/rejected": -0.8359375,
"logps/chosen": -13.8125,
"logps/rejected": -13.875,
"loss": 0.1213923841714859,
"rewards/accuracies": 0.5625,
"rewards/chosen": -133.80438232421875,
"rewards/margins": 0.244720458984375,
"rewards/rejected": -134.04910278320312,
"step": 120
},
{
"epoch": 0.8927038626609443,
"grad_norm": 62.457306149658216,
"ht_mnpo/logit": 0.18063659965991974,
"ht_mnpo/residual": 0.1731366068124771,
"ht_mnpo/residual_abs": 0.3555470108985901,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 2.0489971081290193e-08,
"logits/chosen": -0.78125,
"logits/rejected": -0.80859375,
"logps/chosen": -13.875,
"logps/rejected": -13.9375,
"loss": 0.5861908197402954,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -133.495361328125,
"rewards/margins": 1.806365966796875,
"rewards/rejected": -135.30172729492188,
"step": 130
},
{
"epoch": 0.9613733905579399,
"grad_norm": 59.652930086354274,
"ht_mnpo/logit": 0.009463501162827015,
"ht_mnpo/residual": 0.001963504124432802,
"ht_mnpo/residual_abs": 0.3243860602378845,
"ht_mnpo/target": 0.007499999366700649,
"ht_mnpo/target_abs": 0.007499999366700649,
"learning_rate": 3.5143346177878565e-09,
"logits/chosen": -0.79296875,
"logits/rejected": -0.83203125,
"logps/chosen": -13.875,
"logps/rejected": -13.9375,
"loss": 0.5267720818519592,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -134.2991943359375,
"rewards/margins": 0.094635009765625,
"rewards/rejected": -134.39382934570312,
"step": 140
},
{
"epoch": 1.0,
"eval_ht_mnpo/logit": 0.0946219190955162,
"eval_ht_mnpo/residual": 0.08712193369865417,
"eval_ht_mnpo/residual_abs": 0.23038248717784882,
"eval_ht_mnpo/target": 0.007499998901039362,
"eval_ht_mnpo/target_abs": 0.007499998901039362,
"eval_logits/chosen": -0.78515625,
"eval_logits/rejected": -0.76953125,
"eval_logps/chosen": -13.8125,
"eval_logps/rejected": -13.875,
"eval_loss": 0.09535243362188339,
"eval_rewards/accuracies": 0.6578947305679321,
"eval_rewards/chosen": -134.2300262451172,
"eval_rewards/margins": 0.9462206959724426,
"eval_rewards/rejected": -135.17623901367188,
"eval_runtime": 70.1838,
"eval_samples_per_second": 8.606,
"eval_steps_per_second": 1.083,
"step": 146
},
{
"epoch": 1.0,
"step": 146,
"total_flos": 0.0,
"train_loss": 0.5071899907229698,
"train_runtime": 3280.3544,
"train_samples_per_second": 5.682,
"train_steps_per_second": 0.045
}
],
"logging_steps": 10,
"max_steps": 146,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 1000,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c2c2e246504705393c26a1d7d6037bc4db16586f5468f14d8745f3b477fa8aee
size 8401