From 82e27182615d62794f81faab5d61b6e326a43b1a Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Wed, 6 May 2026 23:43:53 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Aletheia-Bench/DPO-Think-1.5B Source: Original Platform --- .gitattributes | 36 ++++++++ README.md | 55 ++++++++++++ chat_template.jinja | 53 +++++++++++ config.json | 60 +++++++++++++ generation_config.json | 12 +++ model.safetensors | 3 + special_tokens_map.json | 17 ++++ tokenizer.json | 3 + tokenizer_config.json | 194 ++++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 10 files changed, 436 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a393821 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +--- +library_name: transformers +license: cc-by-nc-sa-4.0 +datasets: +- Aletheia-Bench/Aletheia-DPO +base_model: +- deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B +--- + +
+[[**🤗 Model & Dataset**](https://huggingface.co/Aletheia-Bench)] +[[**📊 Code**](https://github.com/insait-institute/aletheia)] +[[**📖 Paper**](https://arxiv.org/)] +
 + +# Aletheia: What Makes RLVR For Code Verifiers Tick? +Multi-domain thinking verifiers trained via Reinforcement Learning from Verifiable Rewards (RLVR) are a prominent fixture of the Large Language Model (LLM) post-training pipeline, owing to their ability to robustly rate and rerank model outputs. However, the adoption of such verifiers towards code generation has been comparatively sparse, with execution feedback constituting the dominant signal. Nonetheless, code verifiers remain valuable toward judging model outputs in scenarios where execution feedback is hard to obtain and are a potentially powerful addition to the code generation post-training toolbox. To this end, we create and open-source Aletheia, a controlled testbed that enables execution-grounded evaluation of code verifiers' robustness across disparate policy models and covariate shifts. We examine components of the RLVR-based verifier training recipe widely credited for its success: (1) intermediate thinking traces, (2) learning from negative samples, and (3) on-policy training. While experiments show the optimality of RLVR, we uncover important opportunities to simplify the recipe. Particularly, despite code verification exhibiting positive training- and inference-time scaling, on-policy learning stands out as the key component at small verifier sizes, and thinking-based training emerges as the most important component at larger scales. + + + +## 📦 Model Zoo +Fine-tuned code verifiers at 1.5B, 7B, and 14B scales using several algorithms +| Algorithm | Thinking | Negatives | Online | Description | +| :--- | :---: | :---: | :---: | :--- | +| **GRPO-Think** | ✅ | ✅ | ✅ | Standard GRPO-style approach to training verifiers. | +| **GRPO-Instruct** | ❌ | ✅ | ✅ | RLVR training without intermediate thinking traces. | +| **RAFT** | ✅ | ❌ | ✅ | On-policy rejection sampling fine-tuning using only positive reasoning samples. | +| **Think-DPO** | ✅ | ✅ | ❌ | Offline preference optimization using pre-collected thinking traces. | +| **Batch-online GRPO** | ✅ | ✅ | ⚠️ | Semi-online training where the generator policy is synced every 4 steps. | + +## 🎁 Datasets +The Aletheia dataset collection is available on HuggingFace and includes: + +* **Aletheia-Train**: A dataset of 2-5 candidate code snippets to solve a coding problem, each with exactly one correct code +* **Aletheia-DPO**: A companion dataset to Aletheia-Train, containing "chosen" and "rejected" responses for each instance. The chosen response identifies the correct code snippet, while the rejected responses do not. +* **Aletheia-Heldout**: A completely in-distribution test set +* **Aletheia-Strong**: An OOD test set where the candidates are generated by stronger models +* **Aletheia-Hard**: An OOD test set where the comparison between candidates is more difficult +* **Aletheia-Adv**: An OOD test set where the candidates are adversarially modified to exploit common LLM biases + +## 💡 Intended uses +* **RLHF / RLAIF**: plug-and-play reward function for code generation policy optimization. +* **Automated evaluation**: LLM-as-a-judge for a variety of code-related tasks. +* **Research**: study the effects of thinking traces, on-policy learning, and negative samples in training successful code verifiers + +## Citations + +```bibtex +@article{aletheia2025, + title={Aletheia: What Makes RLVR For Code Verifiers Tick?}, + author={Venkatkrishna, Vatsal and Paul, Indraneil and Gurevych, Iryna}, + journal={arXiv preprint}, + year={2025} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..ef6dbbe --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,53 @@ +{%- if not add_generation_prompt is defined -%} + {%- set add_generation_prompt = false -%} +{%- endif -%} +{%- set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt="") -%} +{%- for message in messages -%} + {%- if message["role"] == "system" -%} + {%- set ns.system_prompt = message["content"] -%} + {%- endif -%} +{%- endfor -%} +{{- bos_token -}} +{{- ns.system_prompt -}} +{%- for message in messages -%} + {%- if message["role"] == "user" -%} + {%- set ns.is_tool = false -%} + {{- "<｜User｜>" + message["content"] -}} + {%- endif -%} + {%- if message["role"] == "assistant" and message["content"] is none -%} + {%- set ns.is_tool = false -%} + {%- for tool in message["tool_calls"] -%} + {%- if not ns.is_first -%} + {{- "<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>" + tool["type"] + "<｜tool▁sep｜>" + tool["function"]["name"] + "\n" + "```json" + "\n" + tool["function"]["arguments"] + "\n" + "```" + "<｜tool▁call▁end｜>" -}} + {%- set ns.is_first = true -%} + {%- else -%} + {{- "\n" + "<｜tool▁call▁begin｜>" + tool["type"] + "<｜tool▁sep｜>" + tool["function"]["name"] + "\n" + "```json" + "\n" + tool["function"]["arguments"] + "\n" + "```" + "<｜tool▁call▁end｜>" -}} + {{- "<｜tool▁calls▁end｜><｜end▁of▁sentence｜>" -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- if message["role"] == "assistant" and message["content"] is not none -%} + {%- if ns.is_tool -%} + {{- "<｜tool▁outputs▁end｜>" + message["content"] + "<｜end▁of▁sentence｜>" -}} + {%- set ns.is_tool = false -%} + {%- else -%} + {%- set content = message["content"] -%} + {{- "<｜Assistant｜>" + content -}} + {%- endif -%} + {%- endif -%} + {%- if message["role"] == "tool" -%} + {%- set ns.is_tool = true -%} + {%- if ns.is_output_first -%} + {{- "<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>" + message["content"] + "<｜tool▁output▁end｜>" -}} + {%- set ns.is_output_first = false -%} + {%- else -%} + {{- "\n<｜tool▁output▁begin｜>" + message["content"] + "<｜tool▁output▁end｜>" -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if ns.is_tool -%} + {{- "<｜tool▁outputs▁end｜>" -}} +{%- endif -%} +{%- if add_generation_prompt and not ns.is_tool -%} + {{- "<｜Assistant｜>\n" -}} +{%- endif -%} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..23c51d3 --- /dev/null +++ b/config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151646, + "dtype": "bfloat16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151647, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "4.56.1", + "use_cache": true, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..c5310da --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "_from_model_config": true, + "bos_token_id": 151646, + "do_sample": true, + "eos_token_id": [ + 151643 + ], + "pad_token_id": 151647, + "temperature": 0.6, + "top_p": 0.95, + "transformers_version": "4.56.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..91ea05a --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:147342334d07a85aba4fc9ed36faf2e19592e2d222ea07a1f46fe62495a2841f +size 3554214752 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..61b6067 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,17 @@ +{ + "bos_token": { + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|EOT|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1a2db24 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e20ddafc659ba90242154b55275402edeca0715e5dbb30f56815a4ce081f4893 +size 11422778 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..b74720b --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,194 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "151643": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<｜User｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151645": { + "content": "<｜Assistant｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151646": { + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|EOT|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151648": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<｜begin▁of▁sentence｜>", + "clean_up_tokenization_spaces": false, + "eos_token": "<｜end▁of▁sentence｜>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 16384, + "pad_token": "<|EOT|>", + "sp_model_kwargs": {}, + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": null, + "use_default_system_prompt": false +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..d044c42 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e3d4ddcc2a45bd5d94381f8b9d618c5fb4e4dad9362f3573aef7435c6296e18 +size 9041