commit 7fda7409b7f3519d1632c26cb93143dd38ddd587 Author: ModelHub XC Date: Fri Aug 14 10:35:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: williyam/redrob-qwen-grpo Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..955e776 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +training_curves.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..29420a9 --- /dev/null +++ b/README.md @@ -0,0 +1,233 @@ +--- +base_model: Qwen/Qwen3-0.6B +datasets: +- custom-redrob-candidates +language: en +library_name: transformers +license: mit +metrics: +- reward +pipeline_tag: text-generation +tags: +- qwen3 +- qwen3-0.6b +- grpo +- reinforcement-learning +- trl +- candidate-ranking +- explainable-ai +- talentry-ai +- redrob +- hackathon +- open-source +--- + +# redrob-qwen-grpo + +> **`Qwen/Qwen3-0.6B` → GRPO-fine-tuned for explainable candidate ranking, under a rule-based reward model (no LLM-as-a-judge).** + +[![Open in Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Live%20demo-talentry--ai-FFD21E)](https://huggingface.co/spaces/williyam/talentry-ai) +[![GitHub](https://img.shields.io/badge/GitHub-talentry--ai-181717?logo=github&logoColor=white)](https://github.com/williyam-m/talentry-ai) +[![License: MIT](https://img.shields.io/badge/license-MIT-22C55E.svg)](https://github.com/williyam-m/talentry-ai/blob/main/LICENSE) + +This is the **open-source side-quest** of the [Talentry-AI](https://github.com/williyam-m/talentry-ai) +submission to the **Redrob × Hack2Skill — India Runs** Data & AI Challenge. + +The base Talentry-AI ranker is fully deterministic and runs with **0 LLM calls**. +This checkpoint exists for anyone who *wants* an LLM-flavoured candidate ranker +that has been trained against the same rule-based rubric Talentry-AI uses to +audit its own decisions. The Talentry-AI submission itself does **not** depend on +this model. + +--- + +## Headline results + +| Metric | Baseline (`Qwen/Qwen3-0.6B`) | `redrob-qwen-grpo` | Δ | +| ----------------------------- | --------------------------- | ------------------ | ------ | +| Mean rule-based reward `[0,1]` | **0.539** | **0.713** | **+0.173** | +| Eval episodes | 12 | 12 | — | +| Hardware | Apple M1 Pro 16 GB · MPS | Apple M1 Pro 16 GB · MPS | — | +| Eval `max_new_tokens` | 384 | 384 | — | + +The same deterministic eval rollout (`seed=0`, sequential, identical prompts) +is used for both rows so the comparison is fair. + +## Per-component improvement (rule-based reward, mean over eval episodes) + +| Reward component | Baseline | Trained | Δ | +| ------------------ | -------- | ------- | -- | +| `format_valid` | 0.833 | 1.000 | +0.167 | +| `decision_match` | 0.500 | 0.500 | +0.000 | +| `score_alignment` | 0.373 | 0.653 | +0.280 | +| `reason_quality` | 0.000 | 0.778 | +0.778 | +| `length_penalty` | 1.000 | 1.000 | +0.000 | +| `no_hallucination` | 0.779 | 0.656 | -0.124 | +| `total` | 0.539 | 0.713 | +0.173 | + +> All components are in `[0, 1]`. `total` is the weighted convex combination +> (see [`reward.py`](https://github.com/williyam-m/talentry-ai/blob/main/redrob-reinforcement-learning/src/redrob_rl/reward.py)). + +--- + +## Quick usage + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +tok = AutoTokenizer.from_pretrained("williyam/redrob-qwen-grpo") +mdl = AutoModelForCausalLM.from_pretrained( + "williyam/redrob-qwen-grpo", dtype=torch.float32 +).eval() + +system = ( + "You are RedRob, an explainable candidate-ranking assistant. " + "Decide whether the candidate should be SHORTLISTED for the role. " + "Respond with a single JSON object: " + '{"decision":"shortlist"|"reject","score":0..1,"reasons":[..]}.' +) +user = ( + "[JOB DESCRIPTION]\n\n\n" + "[CANDIDATE]\n" +) + +prompt = tok.apply_chat_template( + [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + tokenize=False, + add_generation_prompt=True, +) +inputs = tok(prompt, return_tensors="pt") +out = mdl.generate(**inputs, max_new_tokens=512, do_sample=False) +print(tok.decode(out[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True)) +``` + +The model is expected to return: + +```json +{ + "decision": "shortlist" | "reject", + "score": 0.0-1.0, + "reasons": ["short, grounded bullet", "..."] +} +``` + +--- + +## Training summary + +| Aspect | Value | +| ---------------------------- | ---------------------------------------------------------------- | +| Base model | `Qwen/Qwen3-0.6B` (600M params, Qwen3 chat template) | +| Algorithm | GRPO (TRL `GRPOTrainer`) | +| Reward signal | **Rule-based** (no LLM judge): six interpretable components | +| Reward components | `format_valid`, `decision_match`, `score_alignment`, `reason_quality`, `length_penalty`, `no_hallucination` | +| Optimiser steps | 10 (deliberately short — sample-efficient demo on a laptop GPU) | +| `num_generations` | 2 (group size; 2-arm advantage estimate) | +| KL coefficient `β` | 0.04 | +| Learning rate | 5e-6 | +| Sampling temperature / top-p | 1.0 / 0.95 | +| Max completion length | 96 tokens (training); 512 tokens (eval, this card) | +| Hardware | Apple M1 Pro 16 GB · MPS (`bf16=False`, `fp16=False`, `fp32`) | +| Gradient checkpointing | Yes (`use_reentrant=False`) | +| Training wall-clock | ~4.5 minutes for 10 steps | + +Full training config: [`configs/grpo_qwen3_0p6b.yaml`](https://github.com/williyam-m/talentry-ai/blob/main/redrob-reinforcement-learning/configs/grpo_qwen3_0p6b.yaml). + +## Reward model (no LLM judge) + +Every completion is graded by [`RuleBasedRewardModel`](https://github.com/williyam-m/talentry-ai/blob/main/redrob-reinforcement-learning/src/redrob_rl/reward.py) +on six components, each clipped to `[0, 1]`: + +| Component | What it measures | +| ----------------- | ------------------------------------------------------------------------- | +| `format_valid` | Output parses as `{"decision","score","reasons"}` JSON. | +| `decision_match` | Matches gold `"shortlist" / "reject"` label. | +| `score_alignment` | `1 - │pred_score - gold_score│`. | +| `reason_quality` | 2–5 short, diverse reasons that aren't copy-pasted from the input. | +| `length_penalty` | Stays inside a sensible character budget. | +| `no_hallucination`| Proper nouns / numbers in reasons all appear in the JD or candidate text. | + +Total reward = convex combination (weights documented in the dataclass), so +`total ∈ [0, 1]`. + +--- + +## Plots + +The four training plots are committed to this repo and rendered inline below: + +

+ Training curves + Baseline vs trained +

+

+ Reward components + Reward distribution +

+ +| File | Description | +| ----------------------------- | ---------------------------------------------------------------------- | +| `training_curves.png` | Mean reward `[0,1]` (left axis) + GRPO loss (right axis) vs train step.| +| `baseline_vs_trained.png` | Per-episode reward on the same eval rollout, baseline vs trained. | +| `reward_components.png` | Mean value of each rule-based reward component, baseline vs trained. | +| `reward_distribution.png` | Histogram of episode rewards across the eval rollout. | + +--- + +## Intended use + +* **Educational / research** — show how GRPO with a rule-based reward + shapes a small open-source LLM toward a structured JSON output schema + for a real-world hiring-adjacent task. +* **Drop-in component** — for anyone who wants to plug an LLM ranker into + a candidate-shortlisting pipeline and get an auditable JSON `{decision, + score, reasons}` response. +* **Reference implementation** — the entire training loop, env, and reward + model are open-source under MIT + ([source](https://github.com/williyam-m/talentry-ai/tree/main/redrob-reinforcement-learning)). + +## Out-of-scope / limitations + +* **Not a substitute for human review.** This model produces a *score* and + *reasons*; final hiring decisions must always involve a human reviewer. +* **Trained on a 30-sample distilled fixture** of the Redrob hackathon's + candidate pool — it is *not* trained on the full 100K candidate + population and will not generalise to arbitrary new JDs without + fine-tuning on your own data. +* **Short training run** (10 GRPO steps). The reward shapes can move + meaningfully more with longer training; this checkpoint is the + hackathon-submission burst, not a SOTA result. +* **Single-language** (English). +* **Possible biases** inherited from `Qwen/Qwen3-0.6B`'s pre-training data + and from the synthetic dataset of 50 Redrob candidates. +* **Honeypot resistance** is provided by Talentry-AI's deterministic + pipeline, not by this checkpoint — the LLM here cannot, by itself, + detect "8 years at a 3-year-old company"-style impossibilities. + +## Citation + +If you use this checkpoint, please cite: + +```bibtex +@misc{redrob_qwen_grpo_2026, + title = {redrob-qwen-grpo: GRPO fine-tune of Qwen3-0.6B for explainable candidate ranking}, + author = {Williyam M}, + year = {2026}, + url = {https://huggingface.co/williyam/redrob-qwen-grpo}, + note = {Open-source artifact from the Talentry-AI / Redrob × Hack2Skill - India Runs submission.} +} +``` + +## License + +MIT — see the [Talentry-AI LICENSE](https://github.com/williyam-m/talentry-ai/blob/main/LICENSE). + +## Acknowledgements + +* `Qwen/Qwen3-0.6B` from the Qwen team. +* `trl` for the GRPO implementation. +* `Redrob × Hack2Skill — India Runs` for the JD + 50-candidate fixture. diff --git a/baseline_vs_trained.png b/baseline_vs_trained.png new file mode 100644 index 0000000..c95dab3 Binary files /dev/null and b/baseline_vs_trained.png differ diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..29639d6 --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.12.1", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/eval_compare.json b/eval_compare.json new file mode 100644 index 0000000..1823d00 --- /dev/null +++ b/eval_compare.json @@ -0,0 +1,63 @@ +{ + "device": "mps", + "n_episodes": 12, + "eval_seed": 0, + "max_new_tokens": 384, + "baseline": { + "model": "Qwen/Qwen3-0.6B", + "mean_reward": 0.5394954166666667, + "rewards": [ + 0.49674999999999997, + 0.6875500000000001, + 0.05, + 0.54325, + 0.7235050000000001, + 0.5267499999999999, + 0.7247800000000001, + 0.7245250000000001, + 0.6965800000000001, + 0.7235050000000001, + 0.05, + 0.5267499999999999 + ], + "components_mean": { + "format_valid": 0.8333333333333334, + "decision_match": 0.5, + "score_alignment": 0.373025, + "reason_quality": 0.0, + "length_penalty": 1.0, + "no_hallucination": 0.7791666666666667, + "total": 0.5394954166666667 + }, + "wall_seconds": 107.62254476547241 + }, + "trained": { + "model": "williyam/redrob-qwen-grpo", + "mean_reward": 0.7129858675096833, + "rewards": [ + 0.5129642857142858, + 0.9165815789473685, + 0.5111792857142857, + 0.515, + 0.9074595454545454, + 0.51275, + 0.9070771428571428, + 0.9172250000000001, + 0.91517, + 0.9152192857142857, + 0.51224, + 0.5129642857142858 + ], + "components_mean": { + "format_valid": 1.0, + "decision_match": 0.5, + "score_alignment": 0.6529333333333334, + "reason_quality": 0.7780835611756665, + "length_penalty": 1.0, + "no_hallucination": 0.6555555555555556, + "total": 0.7129858675096833 + }, + "wall_seconds": 67.35159301757812 + }, + "uplift": 0.17349045084301662 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..0b4efd3 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.12.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..67c96e5 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63de169be55d0f994c9363ac71c4bcbeb33561f6d7f578502890c5785fdee906 +size 2384234968 diff --git a/reward_components.png b/reward_components.png new file mode 100644 index 0000000..382df75 Binary files /dev/null and b/reward_components.png differ diff --git a/reward_distribution.png b/reward_distribution.png new file mode 100644 index 0000000..1673c09 Binary files /dev/null and b/reward_distribution.png differ diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..f9c5aac --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e946ac23b6a68f7a2abbe7b3c22190673c6d3d159b85305268db51b2729ac68a +size 11422749 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_curves.png b/training_curves.png new file mode 100644 index 0000000..06d11b9 --- /dev/null +++ b/training_curves.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ca4075f513c1b0472f3266dd6d95947fd68df4609456de06d6246f03ccce9c7 +size 114237